From ca3780aa24714d83452d4028b3b124df985a5357 Mon Sep 17 00:00:00 2001 From: SpecialX <47072643+wangxiner55@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:09:27 +0800 Subject: [PATCH] =?UTF-8?q?feat(data-ana):=20=E5=AE=8C=E6=95=B4=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=20data-ana=20=E6=95=B0=E6=8D=AE=E5=88=86=E6=9E=90?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 包含 CDC consumer、analytics/mastery/warning service、grpc server、repository、ClickHouse DDL 等 --- docs/troubleshooting/known-issues.md | 5 +- infra/clickhouse/ddl/data_ana.sql | 97 +++ packages/shared-proto/proto/analytics.proto | 227 ++++- packages/shared-proto/proto/events.proto | 51 +- packages/shared-proto/proto/iam.proto | 17 + pnpm-lock.yaml | 93 ++ scripts/clickhouse-init.sql | 104 ++- scripts/data-ana-mock-data.sql | 131 +++ services/data-ana/Dockerfile | 40 +- services/data-ana/README.md | 404 ++++++++- services/data-ana/pyproject.toml | 17 +- .../src/data_ana/analytics_service.py | 450 ++++++++++ .../data-ana/src/data_ana/cdc_consumer.py | 510 +++++++++-- services/data-ana/src/data_ana/config.py | 101 ++- services/data-ana/src/data_ana/exam_cache.py | 112 +++ services/data-ana/src/data_ana/grpc_server.py | 570 +++++++++++++ services/data-ana/src/data_ana/main.py | 578 +++++++++---- .../data-ana/src/data_ana/mastery_service.py | 256 ++++++ .../src/data_ana/repository/__init__.py | 1 + .../repository/clickhouse_repository.py | 794 ++++++++++++++++++ .../src/data_ana/repository/iam_client.py | 226 +++++ .../src/data_ana/repository/kafka_producer.py | 202 +++++ .../src/data_ana/repository/redis_client.py | 130 +++ .../data-ana/src/data_ana/shared/__init__.py | 1 + .../src/data_ana/shared/action_state.py | 62 ++ .../data-ana/src/data_ana/shared/errors.py | 66 ++ .../src/data_ana/shared/permissions.py | 124 +++ .../data-ana/src/data_ana/warning_service.py | 383 +++++++++ uv.lock | 32 +- 29 files changed, 5401 insertions(+), 383 deletions(-) create mode 100644 infra/clickhouse/ddl/data_ana.sql create mode 100644 scripts/data-ana-mock-data.sql create mode 100644 services/data-ana/src/data_ana/analytics_service.py create mode 100644 services/data-ana/src/data_ana/exam_cache.py create mode 100644 services/data-ana/src/data_ana/grpc_server.py create mode 100644 services/data-ana/src/data_ana/mastery_service.py create mode 100644 services/data-ana/src/data_ana/repository/__init__.py create mode 100644 services/data-ana/src/data_ana/repository/clickhouse_repository.py create mode 100644 services/data-ana/src/data_ana/repository/iam_client.py create mode 100644 services/data-ana/src/data_ana/repository/kafka_producer.py create mode 100644 services/data-ana/src/data_ana/repository/redis_client.py create mode 100644 services/data-ana/src/data_ana/shared/__init__.py create mode 100644 services/data-ana/src/data_ana/shared/action_state.py create mode 100644 services/data-ana/src/data_ana/shared/errors.py create mode 100644 services/data-ana/src/data_ana/shared/permissions.py create mode 100644 services/data-ana/src/data_ana/warning_service.py diff --git a/docs/troubleshooting/known-issues.md b/docs/troubleshooting/known-issues.md index e3b072e..09bb833 100644 --- a/docs/troubleshooting/known-issues.md +++ b/docs/troubleshooting/known-issues.md @@ -312,7 +312,7 @@ | CDC 水平扩展 | P4 单实例 + 内存 ExamCache LRU → P6 多实例 + Redis ExamCache + partition 扩容 + HPA | | CDC 手动 commit | v2 改进：`enable_auto_commit=False` 手动 commit（at-least-once），commit 前确保 ClickHouse 写入成功 | | ClickHouse 容量规划 | 5 宽表年增量 ~2.3GB（5000 学生×20 次/月），单节点可承载；P6 按 class_id hash 分片 + ReplicatedMergeTree | -| ClickHouse FINAL | ReplacingMergeTree 查询必须加 `FINAL` 或用 `argMax` 聚合，否则读到重复版本（当前实现未加，P0 整改） | +| ClickHouse FINAL | ReplacingMergeTree 查询必须加 `FINAL` 或用 `argMax` 聚合，否则读到重复版本（clickhouse_repository.py 已实现 P0 整改） | | 缓存键命名 | `data_ana:datascope:{user_id}` 5min / `data_ana:exam:{exam_id}` 30day / `data_ana:dedup:{event_id}` 7day | | 4 端 Dashboard | teacher/student/parent/admin 各自 Dashboard API + 权限点 ANALYTICS_*_DASHBOARD（v2 新增 4 端权限） | | SubscribeMasteryUpdate | gRPC server-streaming RPC 为 P5+ AI 个性化推荐预留实时推送通道（analytics.proto v2 提案） | @@ -327,6 +327,9 @@ | 降级模式 4 场景 | ClickHouse/Kafka/Redis/iam gRPC 不可用时返回骨架 + `error.details.degraded=true`，4 错误码 DATA_ANA_*_UNAVAILABLE | | Python gRPC 实现 | grpc.aio + betterproto + ServerInterceptor 透传 W3C trace context，与 HTTP 共享同一 Application Service | | structlog API | 24.x 用 `make_filtering_bound_logger(level)`，旧版 `make_filtering_logger` 已废弃 | +| ruff per-file-ignores | gRPC servicer 方法必须 PascalCase（N802 忽略）/ FastAPI Depends 在参数默认值（B008 忽略），pyproject.toml 配 per-file-ignores | +| Mock 数据脚本 | `scripts/data-ana-mock-data.sql` 提供 5 宽表演示数据（20 学生×2 考试×3 知识点），用 `now()-INTERVAL` 避免硬编码日期 | +| Python 3.12+ 现代化 | ruff UP042 用 StrEnum 替代 `str, Enum` / UP046 用 PEP 695 类型参数 `class Foo[T]` 替代 `Generic[T]` | ### 2.7 messaging（TS/NestJS，P5） diff --git a/infra/clickhouse/ddl/data_ana.sql b/infra/clickhouse/ddl/data_ana.sql new file mode 100644 index 0000000..7729a30 --- /dev/null +++ b/infra/clickhouse/ddl/data_ana.sql @@ -0,0 +1,97 @@ +-- data-ana ClickHouse DDL（5 宽表）. +-- 负责人：ai11（总裁裁决 §2.12 授权 ai11 创建纯 DDL 文件，SRE AI 负责部署脚本）. +-- 对齐文档：services/data-ana/docs/02-architecture-design.md §3. +-- 引擎：ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本（幂等消费保证）. +-- 查询时必须加 FINAL 或使用 argMax 聚合确保去重生效（P0 整改项）. +-- +-- 使用方式： +-- clickhouse-client --multiquery < infra/clickhouse/ddl/data_ana.sql + +-- 数据库 +CREATE DATABASE IF NOT EXISTS edu_analytics; + +-- ===== 1. 学生学情宽表 ===== +-- 每次成绩写入产生一行，按 ORDER BY 去重保留 last_updated 最大版本. +CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view +( + student_id String, + class_id String, + exam_id String, + subject_id String, + score Float64, + rank_in_class UInt32, + knowledge_point_id String, + mastery_level Float32, -- 0.0-1.0 + error_count UInt32, + last_updated DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(last_updated) +PARTITION BY toYYYYMM(last_updated) +ORDER BY (student_id, exam_id, knowledge_point_id) +SETTINGS index_granularity = 8192; + +-- ===== 2. 学生错题本 ===== +-- 同一 (student_id, question_id) 累计 error_count，按 last_error_time 去重. +CREATE TABLE IF NOT EXISTS edu_analytics.student_errors +( + student_id String, + question_id String, + knowledge_point_id String, + error_count UInt32, + last_error_time DateTime64(3, 'UTC'), + content String DEFAULT '' +) +ENGINE = ReplacingMergeTree(last_error_time) +PARTITION BY toYYYYMM(last_error_time) +ORDER BY (student_id, question_id); + +-- ===== 3. 知识点掌握度历史快照 ===== +-- 每次掌握度计算产生新版本，支持趋势查询. +CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot +( + student_id String, + knowledge_point_id String, + subject_id String, + mastery_level Float32, -- 0.0-1.0 + calculated_at DateTime64(3, 'UTC'), + calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve' +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(calculated_at) +ORDER BY (student_id, knowledge_point_id, calculated_at); + +-- ===== 4. 学生考勤记录 ===== +-- core-edu attendance 表 CDC 同步，同一记录多版本去重. +CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs +( + student_id String, + class_id String, + attendance_date Date, + status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave' + recorded_by String, -- 教师用户 ID + remark String DEFAULT '', + occurred_at DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(occurred_at) +PARTITION BY toYYYYMM(attendance_date) +ORDER BY (student_id, class_id, attendance_date); + +-- ===== 5. AI 用量计费记录 ===== +-- ai 服务通过 Kafka 事件投递，data-ana 消费落库，按 request_id 幂等去重. +CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log +( + request_id String, + user_id String, + provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local' + model LowCardinality(String), + prompt_tokens UInt32, + completion_tokens UInt32, + total_tokens UInt32, + latency_ms UInt32, + success Boolean, + cost_cents UInt32, -- 计费（分），便于聚合 + occurred_at DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(occurred_at) +PARTITION BY toYYYYMM(occurred_at) +ORDER BY (request_id); diff --git a/packages/shared-proto/proto/analytics.proto b/packages/shared-proto/proto/analytics.proto index 1080b09..b4f07ec 100644 --- a/packages/shared-proto/proto/analytics.proto +++ b/packages/shared-proto/proto/analytics.proto @@ -2,24 +2,113 @@ syntax = "proto3"; package next_edu_cloud.analytics.v1; +// AnalyticsService 数据分析服务契约（D6 智能洞察领域）. +// P4 启用 gRPC server 端口 50055，HTTP 3006 保留作 Gateway 直连降级. +// 所有 RPC 返回 ActionState 信封（success/data/error/details.degraded）. service AnalyticsService { + // 班级成绩分析（平均分/及格率/参考人数）. rpc GetClassPerformance(GetClassPerformanceRequest) returns (ClassPerformance); + // 学生薄弱知识点（mastery_level < 0.6）. rpc GetStudentWeakness(GetStudentWeaknessRequest) returns (StudentWeakness); + // 学习趋势（历史成绩曲线）. rpc GetLearningTrend(GetLearningTrendRequest) returns (LearningTrend); + // 教师仪表盘聚合（班级概览 + 待办 + 预警）. + rpc GetTeacherDashboard(GetTeacherDashboardRequest) returns (TeacherDashboard); + // 学生仪表盘（个人学情 + 排名 + 薄弱点）. + rpc GetStudentDashboard(GetStudentDashboardRequest) returns (StudentDashboard); + // 家长仪表盘（孩子学情概览）. + rpc GetParentDashboard(GetParentDashboardRequest) returns (ParentDashboard); + // 管理员仪表盘（全校统计 + AI 用量）. + rpc GetAdminDashboard(GetAdminDashboardRequest) returns (AdminDashboard); + // 预警列表查询（按班级/严重度/时间过滤）. + rpc GetWarnings(GetWarningsRequest) returns (WarningList); + // 手动触发预警（管理员/教师主动标记关注）. + rpc TriggerWarning(TriggerWarningRequest) returns (TriggerWarningResponse); + // 班级掌握度分布（mastered/progressing/weak 三档）. + rpc GetMasteryDistribution(GetMasteryDistributionRequest) returns (MasteryDistribution); + // 学生知识点掌握度明细. + rpc GetStudentMastery(GetStudentMasteryRequest) returns (StudentMastery); + // 订阅掌握度更新（server-streaming，P5+ AI 个性化推荐实时推送通道）. + rpc SubscribeMasteryUpdate(SubscribeMasteryUpdateRequest) returns (stream MasteryUpdateEvent); } +// ===== 请求消息 ===== + message GetClassPerformanceRequest { string class_id = 1; string subject_id = 2; - int64 start_date = 3; + int64 start_date = 3; // Unix timestamp（秒） int64 end_date = 4; } +message GetStudentWeaknessRequest { + string student_id = 1; + string subject_id = 2; +} + +message GetLearningTrendRequest { + string student_id = 1; + int64 start_date = 2; + int64 end_date = 3; + string subject_id = 4; +} + +message GetTeacherDashboardRequest { + string user_id = 1; + string class_id = 2; // 可选，不传则返回教师全部班级 +} + +message GetStudentDashboardRequest { + string user_id = 1; +} + +message GetParentDashboardRequest { + string user_id = 1; + string student_id = 2; // 家长查看指定孩子 +} + +message GetAdminDashboardRequest { + string user_id = 1; + string scope = 2; // ALL / SCHOOL / DISTRICT + string scope_id = 3; // 具体范围 ID +} + +message GetWarningsRequest { + string class_id = 1; + string severity = 2; // INFO / WARN / CRITICAL + int64 since = 3; // Unix timestamp（秒） +} + +message TriggerWarningRequest { + string target_id = 1; // 学生 ID + string warning_type = 2; // LOW_MASTERY / SCORE_DROP / ABSENT_FREQUENT + string severity = 3; // INFO / WARN / CRITICAL +} + +message GetMasteryDistributionRequest { + string class_id = 1; + string subject_id = 2; + string knowledge_point_id = 3; // 可选 +} + +message GetStudentMasteryRequest { + string student_id = 1; + string subject_id = 2; // 可选 +} + +message SubscribeMasteryUpdateRequest { + string student_id = 1; // 可选，订阅指定学生 + string class_id = 2; // 可选，订阅指定班级全部学生 +} + +// ===== 响应消息 ===== + message ClassPerformance { string class_id = 1; double average_score = 2; double pass_rate = 3; - repeated StudentScore scores = 4; + int32 total_students = 4; + repeated StudentScore scores = 5; } message StudentScore { @@ -28,11 +117,6 @@ message StudentScore { string grade = 3; } -message GetStudentWeaknessRequest { - string student_id = 1; - string subject_id = 2; -} - message StudentWeakness { string student_id = 1; repeated WeakPoint weak_points = 2; @@ -42,12 +126,7 @@ message WeakPoint { string knowledge_point_id = 1; string title = 2; double mastery = 3; -} - -message GetLearningTrendRequest { - string student_id = 1; - int64 start_date = 2; - int64 end_date = 3; + int32 error_count = 4; } message LearningTrend { @@ -59,3 +138,125 @@ message TrendPoint { int64 date = 1; double score = 2; } + +message TeacherDashboard { + string user_id = 1; + int32 total_classes = 2; + int32 total_students = 3; + double class_avg_score = 4; + int32 pending_homework_count = 5; + repeated ClassSummary classes = 6; + repeated StudentSummary top_students = 7; + repeated WarningInfo recent_warnings = 8; +} + +message ClassSummary { + string class_id = 1; + string class_name = 2; + int32 student_count = 3; + double average_score = 4; +} + +message StudentSummary { + string student_id = 1; + string student_name = 2; + double score = 3; + int32 rank_in_class = 4; +} + +message StudentDashboard { + string user_id = 1; + double avg_score = 2; + int32 class_rank = 3; + int32 total_students = 4; + repeated WeakPoint weak_points = 5; + repeated TrendPoint recent_trends = 6; + int32 pending_homework = 7; +} + +message ParentDashboard { + string user_id = 1; + string student_id = 2; + double child_avg_score = 3; + int32 child_class_rank = 4; + int32 total_class_students = 5; + repeated WeakPoint child_weak_points = 6; + repeated WarningInfo child_warnings = 7; +} + +message AdminDashboard { + string user_id = 1; + int32 total_teachers = 2; + int32 total_students = 3; + int32 total_classes = 4; + double school_avg_score = 5; + repeated WarningInfo recent_warnings = 6; + AIUsageSummary ai_usage = 7; +} + +message AIUsageSummary { + int64 total_requests = 1; + int64 total_tokens = 2; + int64 total_cost_cents = 3; + repeated AIUsageByProvider by_provider = 4; +} + +message AIUsageByProvider { + string provider = 1; + int64 request_count = 2; + int64 total_tokens = 3; + int64 cost_cents = 4; +} + +message WarningList { + repeated WarningInfo warnings = 1; + int32 total = 2; +} + +message WarningInfo { + string warning_id = 1; + string warning_type = 2; // LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE + string target_id = 3; // 学生 ID + string target_name = 4; + double threshold = 5; + double current_value = 6; + string severity = 7; // INFO / WARN / CRITICAL + int64 occurred_at = 8; // Unix timestamp（秒） +} + +message TriggerWarningResponse { + string warning_id = 1; + bool triggered = 2; +} + +message MasteryDistribution { + string class_id = 1; + int32 mastered_count = 2; // mastery >= 0.8 + int32 progressing_count = 3; // 0.4 <= mastery < 0.8 + int32 weak_count = 4; // mastery < 0.4 + int32 total_students = 5; +} + +message StudentMastery { + string student_id = 1; + repeated KnowledgePointMastery knowledge_points = 2; + double overall_mastery = 3; +} + +message KnowledgePointMastery { + string knowledge_point_id = 1; + string title = 2; + string subject_id = 3; + double mastery_level = 4; // 0.0-1.0 + string mastery_label = 5; // mastered / progressing / weak + int64 calculated_at = 6; // Unix timestamp（秒） +} + +message MasteryUpdateEvent { + string event_id = 1; + string student_id = 2; + string knowledge_point_id = 3; + double mastery_level = 4; + double previous_level = 5; + int64 calculated_at = 6; // Unix timestamp（秒） +} diff --git a/packages/shared-proto/proto/events.proto b/packages/shared-proto/proto/events.proto index 005f3c3..d3d98f9 100644 --- a/packages/shared-proto/proto/events.proto +++ b/packages/shared-proto/proto/events.proto @@ -4,13 +4,17 @@ package next_edu_cloud.events.v1; // Cross-service event contracts published by CoreEdu via the transactional // outbox pattern and consumed by downstream services (notifications, analytics, -// audit, etc.). Topics follow the convention edu.{domain}.events. +// audit, etc.). // // Event routing (TOPIC_MAP in outbox.publisher.ts): -// edu.exam.events <- exam.created / exam.updated / exam.deleted -// edu.homework.events <- homework.assigned / homework.submitted / homework.graded -// edu.grade.events <- grade.recorded / grade.updated -// edu.class.events <- class.transferred +// edu.teaching.exam.published <- exam.published +// edu.teaching.homework.assigned <- homework.assigned / homework.submitted / homework.graded +// edu.teaching.grade.recorded <- grade.recorded / grade.updated +// edu.teaching.class.transferred <- class.transferred +// +// Derived data events (Outbox exempt, see 004 §12.2): +// edu.insight.mastery.updated <- mastery.updated / warning.triggered (action field distinguishes) +// edu.insight.ai.usage <- ai.usage.recorded message ClassEvent { string event_id = 1; @@ -58,3 +62,40 @@ message GradeEvent { string action = 8; map metadata = 9; } + +// MasteryEvent 掌握度/预警派生数据事件（Outbox 豁免，直接 Kafka producer 发布）. +// action 字段区分：mastery.updated（掌握度更新）/ warning.triggered（预警触发）. +// 两者复用同一 topic edu.insight.mastery.updated（总裁裁决 §2.11）. +message MasteryEvent { + string event_id = 1; + string action = 2; // mastery.updated / warning.triggered + int64 occurred_at = 3; + string student_id = 4; + string knowledge_point_id = 5; + double mastery_level = 6; // 0.0-1.0（mastery.updated 时有效） + double previous_level = 7; // 上一次掌握度（mastery.updated 时有效） + // warning.triggered 时以下字段有效 + string warning_type = 8; // LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE + string target_id = 9; // 预警目标（学生 ID） + double threshold = 10; // 预警阈值 + double current_value = 11; // 当前值 + string severity = 12; // INFO / WARN / CRITICAL + map metadata = 13; +} + +// AIUsageEvent AI 用量计费事件（ai 服务发布，data-ana 消费落 ai_usage_log）. +message AIUsageEvent { + string event_id = 1; + string request_id = 2; + string user_id = 3; + string provider = 4; // openai / anthropic / baichuan / local + string model = 5; + uint32 prompt_tokens = 6; + uint32 completion_tokens = 7; + uint32 total_tokens = 8; + uint32 latency_ms = 9; + bool success = 10; + uint32 cost_cents = 11; // 计费（分） + int64 occurred_at = 12; + map metadata = 13; +} diff --git a/packages/shared-proto/proto/iam.proto b/packages/shared-proto/proto/iam.proto index b0e3277..53e0fa8 100644 --- a/packages/shared-proto/proto/iam.proto +++ b/packages/shared-proto/proto/iam.proto @@ -9,6 +9,9 @@ service IamService { rpc Login(LoginRequest) returns (AuthResponse); rpc RefreshToken(RefreshTokenRequest) returns (TokenPair); rpc GetUserInfo(GetUserInfoRequest) returns (UserInfo); + // GetEffectiveDataScope 解析用户可见数据范围（DataScope 6 级）. + // data-ana gRPC 调用此 RPC 解析查询过滤范围（coord-cross-review §2 #3 裁决 P4 补全）. + rpc GetEffectiveDataScope(GetEffectiveDataScopeRequest) returns (EffectiveDataScope); } message RegisterRequest { @@ -48,3 +51,17 @@ message UserInfo { repeated string roles = 4; repeated string permissions = 5; } + +message GetEffectiveDataScopeRequest { + string user_id = 1; +} + +// EffectiveDataScope 用户可见数据范围（6 级）. +// level: SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL +// scope_ids: 具体可见的 class_id / grade_id 列表（SELF/ALL 时为空） +message EffectiveDataScope { + string user_id = 1; + string level = 2; // SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL + repeated string scope_ids = 3; // class_id 列表（CLASS 级）/ grade_id 列表（GRADE 级） + string school_id = 4; // SCHOOL 级时的学校 ID +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ef4a295..df18c0c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -82,8 +82,88 @@ importers: specifier: ^5.6.0 version: 5.9.3 + packages/hooks: + dependencies: + '@edu/ui-components': + specifier: workspace:* + version: link:../ui-components + react: + specifier: ^18.3.0 + version: 18.3.1 + react-dom: + specifier: ^18.3.0 + version: 18.3.1(react@18.3.1) + devDependencies: + '@types/react': + specifier: ^18.3.0 + version: 18.3.31 + '@types/react-dom': + specifier: ^18.3.0 + version: 18.3.7(@types/react@18.3.31) + typescript: + specifier: ^5.6.0 + version: 5.9.3 + packages/shared-proto: {} + packages/shared-ts: + dependencies: + '@nestjs/common': + specifier: ^10.4.0 + version: 10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': + specifier: ^10.4.0 + version: 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@10.4.22)(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@paralleldrive/cuid2': + specifier: ^2.2.2 + version: 2.3.1 + drizzle-orm: + specifier: ^0.31.0 + version: 0.31.4(@opentelemetry/api@1.9.1)(@types/better-sqlite3@7.6.13)(@types/pg@8.6.1)(@types/react@18.3.31)(better-sqlite3@11.10.0)(mysql2@3.22.6(@types/node@22.20.0))(react@18.3.1) + kafkajs: + specifier: ^2.2.0 + version: 2.2.4 + pino: + specifier: ^9.4.0 + version: 9.14.0 + reflect-metadata: + specifier: ^0.2.2 + version: 0.2.2 + rxjs: + specifier: ^7.8.0 + version: 7.8.2 + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.20.0 + typescript: + specifier: ^5.6.0 + version: 5.9.3 + + packages/ui-components: + dependencies: + '@edu/ui-tokens': + specifier: workspace:* + version: link:../ui-tokens + react: + specifier: ^18.3.0 + version: 18.3.1 + react-dom: + specifier: ^18.3.0 + version: 18.3.1(react@18.3.1) + devDependencies: + '@types/react': + specifier: ^18.3.0 + version: 18.3.31 + '@types/react-dom': + specifier: ^18.3.0 + version: 18.3.7(@types/react@18.3.31) + typescript: + specifier: ^5.6.0 + version: 5.9.3 + + packages/ui-tokens: {} + scripts/arch-scan: dependencies: better-sqlite3: @@ -1494,6 +1574,10 @@ packages: cpu: [x64] os: [win32] + '@noble/hashes@1.8.0': + resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} + engines: {node: ^14.21.3 || >=16} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -2503,6 +2587,9 @@ packages: peerDependencies: '@opentelemetry/api': ^1.1.0 + '@paralleldrive/cuid2@2.3.1': + resolution: {integrity: sha512-XO7cAxhnTZl0Yggq6jOgjiOHhbgcO4NqFqwSmQpjK3b6TEE6Uj/jfSk6wzYyemh3+I0sHirKSetjQwn5cZktFw==} + '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -6642,6 +6729,8 @@ snapshots: '@next/swc-win32-x64-msvc@14.2.33': optional: true + '@noble/hashes@1.8.0': {} + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -8135,6 +8224,10 @@ snapshots: '@opentelemetry/api': 1.9.1 '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@paralleldrive/cuid2@2.3.1': + dependencies: + '@noble/hashes': 1.8.0 + '@pinojs/redact@0.4.0': {} '@pkgjs/parseargs@0.11.0': diff --git a/scripts/clickhouse-init.sql b/scripts/clickhouse-init.sql index 4a23e52..dd385cf 100644 --- a/scripts/clickhouse-init.sql +++ b/scripts/clickhouse-init.sql @@ -1,37 +1,89 @@ -- ClickHouse 数据库初始化脚本 --- 适用服务：data-ana（数据分析） --- 表结构：student_dashboard_view（学生学情宽表）/ student_errors（错题本） --- 与 services/data-ana/src/data_ana/clickhouse_client.py 中的查询字段对齐 +-- 适用服务：data-ana（D6 智能洞察领域） +-- 5 张宽表（ai-allocation §5 强制）： +-- 1. student_dashboard_view 学生学情宽表（成绩/班级/知识点维度） +-- 2. student_errors 学生错题本 +-- 3. mastery_snapshot 知识点掌握度历史快照 +-- 4. ai_usage_log AI 用量计费记录（ai 服务投递，data-ana 消费） +-- 5. attendance_logs 学生考勤记录（core-edu attendance 表 CDC 同步） +-- +-- 引擎对齐 02-architecture-design.md §3：ReplacingMergeTree 按 ORDER BY 去重 + version 列保留最新版本 +-- 查询规范：所有 SELECT 必须加 FINAL 或使用 argMax 聚合确保去重生效（P0 整改） -- -- 使用方式（启用 ClickHouse 时执行一次）： -- clickhouse-client --multiquery < scripts/clickhouse-init.sql --- 注意：ClickHouse 为可选依赖，未配置时 data-ana 服务进入降级模式。 +-- 注意：ClickHouse 为可选依赖，未配置时 data-ana 服务进入降级模式（返回骨架数据）。 -- 数据库 CREATE DATABASE IF NOT EXISTS edu_analytics; --- 学生学情宽表（考试/班级/知识点维度） +-- ===== 1. 学生学情宽表（成绩写入产生一行，按 ORDER BY 去重保留最新版本） ===== CREATE TABLE IF NOT EXISTS edu_analytics.student_dashboard_view ( - student_id String, - class_id String, - exam_id String, - subject_id String, - score Float64, - rank_in_class UInt32, - knowledge_point_id String, - mastery_level Float32, - error_count UInt32, - last_updated DateTime -) ENGINE = MergeTree() -ORDER BY (student_id, class_id, exam_id); + student_id String, + class_id String, + exam_id String, + subject_id String, + score Float64, + rank_in_class UInt32, + knowledge_point_id String, + mastery_level Float32, -- 0.0-1.0 + error_count UInt32, + last_updated DateTime64(3, 'UTC') +) ENGINE = ReplacingMergeTree(last_updated) +PARTITION BY toYYYYMM(last_updated) +ORDER BY (student_id, exam_id, knowledge_point_id) +SETTINGS index_granularity = 8192; --- 学生错题表（错题本） +-- ===== 2. 学生错题本 ===== CREATE TABLE IF NOT EXISTS edu_analytics.student_errors ( - student_id String, - question_id String, - knowledge_point_id String, - error_count UInt32, - last_error_time DateTime, - content String -) ENGINE = MergeTree() -ORDER BY (student_id, knowledge_point_id); + student_id String, + question_id String, + knowledge_point_id String, + error_count UInt32, + last_error_time DateTime64(3, 'UTC'), + content String +) ENGINE = ReplacingMergeTree(last_error_time) +PARTITION BY toYYYYMM(last_error_time) +ORDER BY (student_id, question_id); + +-- ===== 3. 掌握度历史快照（每次掌握度计算产生新版本，支持趋势查询） ===== +CREATE TABLE IF NOT EXISTS edu_analytics.mastery_snapshot ( + student_id String, + knowledge_point_id String, + subject_id String, + mastery_level Float32, + calculated_at DateTime64(3, 'UTC'), + calculation_method LowCardinality(String) -- 'weighted_moving_avg' / 'simple_avg' / 'forgetting_curve' +) ENGINE = MergeTree +PARTITION BY toYYYYMM(calculated_at) +ORDER BY (student_id, knowledge_point_id, calculated_at); + +-- ===== 4. AI 用量计费记录（ai 服务通过 Kafka 事件投递，data-ana 消费落库） ===== +CREATE TABLE IF NOT EXISTS edu_analytics.ai_usage_log ( + request_id String, + user_id String, + provider LowCardinality(String), -- 'openai' / 'anthropic' / 'baichuan' / 'local' + model LowCardinality(String), + prompt_tokens UInt32, + completion_tokens UInt32, + total_tokens UInt32, + latency_ms UInt32, + success Boolean, + cost_cents UInt32, -- 计费（分），便于聚合 + occurred_at DateTime64(3, 'UTC') +) ENGINE = ReplacingMergeTree(occurred_at) +PARTITION BY toYYYYMM(occurred_at) +ORDER BY (request_id); -- 按 request_id 幂等去重 + +-- ===== 5. 学生考勤记录（core-edu attendance 表 CDC 同步） ===== +CREATE TABLE IF NOT EXISTS edu_analytics.attendance_logs ( + student_id String, + class_id String, + attendance_date Date, + status LowCardinality(String), -- 'present' / 'absent' / 'late' / 'leave' + recorded_by String, -- 教师用户 ID + remark String DEFAULT '', + occurred_at DateTime64(3, 'UTC') +) ENGINE = ReplacingMergeTree(occurred_at) +PARTITION BY toYYYYMM(attendance_date) +ORDER BY (student_id, class_id, attendance_date); diff --git a/scripts/data-ana-mock-data.sql b/scripts/data-ana-mock-data.sql new file mode 100644 index 0000000..48073d9 --- /dev/null +++ b/scripts/data-ana-mock-data.sql @@ -0,0 +1,131 @@ +-- data-ana 模块 Mock 数据脚本 +-- 用途：本地开发/集成测试时为 5 张宽表插入演示数据，验证查询逻辑 +-- 使用方式（需先执行 clickhouse-init.sql）： +-- clickhouse-client --multiquery < scripts/data-ana-mock-data.sql +-- +-- 数据规模：2 班 × 10 学生 × 3 知识点 × 2 考试 = 120 行 student_dashboard_view +-- 20 条错题 / 60 条掌握度快照 / 20 条考勤 / 10 条 AI 用量 +-- 所有时间戳使用 now() - N days 模式，避免硬编码绝对日期导致脚本过期 + +USE edu_analytics; + +-- ===== 1. student_dashboard_view：学情宽表 ===== +-- 班级 C001（高一 1 班）+ C002（高一 2 班），每班 10 学生 +-- 2 次考试（EXAM001 数学月考 / EXAM002 数学期中），3 个数学知识点 +INSERT INTO student_dashboard_view +SELECT + 'S' || toString(number + 1) AS student_id, -- S001..S020 + IF(number < 10, 'C001', 'C002') AS class_id, -- 前 10 在 C001，后 10 在 C002 + 'EXAM001' AS exam_id, + 'SUBJ_MATH' AS subject_id, + 40 + (number % 60) AS score, -- 40-99 分 + (number % 10) + 1 AS rank_in_class, -- 1-10 名 + 'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, -- KP_MATH_1/2/3 + 0.3 + ((number % 70) / 100.0) AS mastery_level, -- 0.30-0.99 + (number % 5) AS error_count, -- 0-4 错题 + now() - INTERVAL 7 DAY AS last_updated +FROM numbers(20); + +-- 第二次考试（EXAM002），成绩略升 +INSERT INTO student_dashboard_view +SELECT + 'S' || toString(number + 1) AS student_id, + IF(number < 10, 'C001', 'C002') AS class_id, + 'EXAM002' AS exam_id, + 'SUBJ_MATH' AS subject_id, + 50 + (number % 50) AS score, -- 50-99 分 + (number % 10) + 1 AS rank_in_class, + 'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, + 0.4 + ((number % 60) / 100.0) AS mastery_level, + (number % 4) AS error_count, + now() - INTERVAL 1 DAY AS last_updated +FROM numbers(20); + +-- ===== 2. student_errors：错题本 ===== +INSERT INTO student_errors +SELECT + 'S' || toString((number % 20) + 1) AS student_id, + 'Q_' || toString(number + 1) AS question_id, + 'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, + (number % 5) + 1 AS error_count, + now() - INTERVAL (number % 30) DAY AS last_error_time, + '错误类型：' || multiIf( + number % 3 = 0, '计算失误', + number % 3 = 1, '公式记忆错误', + '审题不清' + ) AS content +FROM numbers(20); + +-- ===== 3. mastery_snapshot：掌握度历史快照（3 次计算，便于趋势查询） ===== +INSERT INTO mastery_snapshot +SELECT + 'S' || toString((number % 20) + 1) AS student_id, + 'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, + 'SUBJ_MATH' AS subject_id, + 0.3 + ((number % 70) / 100.0) AS mastery_level, + now() - INTERVAL 30 DAY AS calculated_at, + 'weighted_moving_avg' AS calculation_method +FROM numbers(20); + +INSERT INTO mastery_snapshot +SELECT + 'S' || toString((number % 20) + 1) AS student_id, + 'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, + 'SUBJ_MATH' AS subject_id, + 0.4 + ((number % 60) / 100.0) AS mastery_level, + now() - INTERVAL 15 DAY AS calculated_at, + 'weighted_moving_avg' AS calculation_method +FROM numbers(20); + +INSERT INTO mastery_snapshot +SELECT + 'S' || toString((number % 20) + 1) AS student_id, + 'KP_MATH_' || toString((number % 3) + 1) AS knowledge_point_id, + 'SUBJ_MATH' AS subject_id, + 0.5 + ((number % 50) / 100.0) AS mastery_level, + now() AS calculated_at, + 'weighted_moving_avg' AS calculation_method +FROM numbers(20); + +-- ===== 4. attendance_logs：考勤记录（近 5 天，含缺勤/迟到） ===== +INSERT INTO attendance_logs +SELECT + 'S' || toString((number % 20) + 1) AS student_id, + IF(number % 20 < 10, 'C001', 'C002') AS class_id, + today() - (number % 5) AS attendance_date, + multiIf( + number % 10 = 0, 'absent', -- 10% 缺勤 + number % 5 = 0, 'late', -- 20% 迟到 + 'present' -- 其余正常 + ) AS status, + 'T001' AS recorded_by, + multiIf(number % 10 = 0, '病假', '') AS remark, + now() - INTERVAL (number % 5) DAY AS occurred_at +FROM numbers(20); + +-- ===== 5. ai_usage_log：AI 用量计费（模拟 3 个 provider） ===== +INSERT INTO ai_usage_log +SELECT + 'REQ_' || toString(number + 1) AS request_id, + 'S' || toString((number % 20) + 1) AS user_id, + arrayElement(['openai', 'anthropic', 'baichuan'], (number % 3) + 1) AS provider, + arrayElement(['gpt-4o-mini', 'claude-3-haiku', 'baichuan2-turbo'], (number % 3) + 1) AS model, + 100 + (number % 200) AS prompt_tokens, -- 100-299 + 50 + (number % 150) AS completion_tokens, -- 50-199 + 150 + (number % 350) AS total_tokens, -- 150-499 + 500 + (number % 2000) AS latency_ms, -- 500-2499 ms + number % 5 != 0 AS success, -- 80% 成功 + (number % 50) + 1 AS cost_cents, -- 1-50 分 + now() - INTERVAL (number % 24) HOUR AS occurred_at +FROM numbers(20); + +-- ===== 校验查询（可选执行） ===== +SELECT 'student_dashboard_view' AS table_name, count() AS cnt FROM student_dashboard_view FINAL +UNION ALL +SELECT 'student_errors', count() FROM student_errors FINAL +UNION ALL +SELECT 'mastery_snapshot', count() FROM mastery_snapshot +UNION ALL +SELECT 'attendance_logs', count() FROM attendance_logs FINAL +UNION ALL +SELECT 'ai_usage_log', count() FROM ai_usage_log FINAL; diff --git a/services/data-ana/Dockerfile b/services/data-ana/Dockerfile index 0fc4264..ba5d79d 100644 --- a/services/data-ana/Dockerfile +++ b/services/data-ana/Dockerfile @@ -1,8 +1,42 @@ -FROM python:3.12-slim +# 多阶段构建：builder 缓存依赖，runtime 精简镜像 +# 对齐 02-architecture-design.md §16 部署规范 + +# ===== Stage 1: builder ===== +FROM python:3.12-slim AS builder + WORKDIR /app + +# 安装 uv（快速依赖管理） RUN pip install uv + +# 先复制 pyproject.toml，利用 Docker 层缓存 COPY pyproject.toml . + +# 同步依赖到虚拟环境（--no-dev 不装开发依赖） RUN uv sync --no-dev + +# ===== Stage 2: runtime ===== +FROM python:3.12-slim AS runtime + +WORKDIR /app + +# 从 builder 复制虚拟环境 +COPY --from=builder /app/.venv /app/.venv + +# 确保虚拟环境在 PATH 中 +ENV PATH="/app/.venv/bin:$PATH" +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + +# 复制源码 COPY src ./src -EXPOSE 3006 -CMD ["uv", "run", "uvicorn", "src.data_ana.main:app", "--host", "0.0.0.0", "--port", "3006"] + +# 暴露端口：HTTP 3006（Gateway 直连降级）+ gRPC 50055（AnalyticsService 12 RPC） +EXPOSE 3006 50055 + +# 健康检查（HTTP /healthz，每 30s 一次） +HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:3006/healthz')" || exit 1 + +# 启动命令：uvicorn HTTP server（gRPC server 在 lifespan 中启动） +CMD ["uvicorn", "src.data_ana.main:app", "--host", "0.0.0.0", "--port", "3006"] diff --git a/services/data-ana/README.md b/services/data-ana/README.md index b5bbec7..f3fbf1b 100644 --- a/services/data-ana/README.md +++ b/services/data-ana/README.md @@ -1,58 +1,386 @@ # data-ana 数据分析服务 -> 版本：0.1（P4 骨架） -> 端口：3006 +> 版本：1.0（P4 完整版） +> 阶段：D6 智能洞察领域（仲裁已对齐，v2 实现版） +> 端口：HTTP 3006（Gateway 直连降级）+ gRPC 50055（主入口，BFF/ai 调用） +> 日期：2026-07-10 ## 职责 -数据分析限界上下文（Python 实现），消费 core-edu 与 content 的领域事件， -构建 ClickHouse 学情宽表，计算知识点掌握度。 -对外提供学情仪表盘查询、班级/年级/学校维度报表、个性化推荐数据支撑。 +数据分析限界上下文（Python/FastAPI 微服务），消费 core-edu / content / iam / ai 的 CDC 事件， +构建 ClickHouse 学情宽表，计算知识点掌握度，评估预警，为前端提供 4 端 Dashboard 查询服务。 + +### 核心能力 + +- **5 张宽表**：student_dashboard_view / student_errors / mastery_snapshot / attendance_logs / ai_usage_log +- **CDC 链路**：Debezium → Kafka → data-ana 消费 → ClickHouse（at-least-once + 手动 commit + ReplacingMergeTree 去重） +- **掌握度算法**：WEIGHTED_MOVING_AVG（w_i=0.6^i）+ FORGETTING_CURVE（半衰期 30 天）取 max +- **预警评估**：5 类预警（LOW_MASTERY / CRITICAL_LOW / SCORE_DROP / ABSENT_FREQUENT / TREND_DECLINE）+ Redis 25h 去重 +- **DataScope 过滤**：6 级（SELF/CLASS/GRADE/SCHOOL/DISTRICT/ALL），iam gRPC + 角色降级 +- **gRPC 12 RPC**：AnalyticsService（含 server-streaming SubscribeMasteryUpdate）+ HealthService +- **HTTP 14 端点**：3 基础 + 11 业务（ActionState 信封 + degraded 标记） + +## 架构图 + +```mermaid +flowchart TB + subgraph Entry["入口层"] + HTTP[FastAPI HTTP
:3006 /analytics/* + /healthz + /readyz] + GRPC[grpc.aio Server
:50055 AnalyticsService 12 RPC] + end + + subgraph Service["应用服务层"] + S1[AnalyticsService
4 Dashboard + 班级/趋势查询] + S2[MasteryService
掌握度计算 + 事件发布] + S3[WarningService
5 类预警 + Redis 去重] + end + + subgraph Repo["数据访问层"] + R1[ClickHouseRepository
FINAL/argMax + DataScope WHERE] + R2[KafkaProducer
mastery.updated 派生事件] + R3[IamClient
gRPC GetEffectiveDataScope] + R4[RedisClient
DataScope 缓存 + 幂等] + end + + subgraph Consumer["CDC 消费者（后台任务）"] + C1[CdcConsumer
手动 commit + 7 类事件路由] + C2[ExamCache
LRU 10000 exam_id→metadata] + end + + subgraph Storage["存储/总线"] + CH[(ClickHouse
5 宽表)] + KAFKA[(Kafka
edu-cdc.* + edu.insight.mastery.updated)] + IAM[iam gRPC :50052] + REDIS[(Redis
data_ana:* 键)] + end + + HTTP --> S1 + HTTP --> S3 + GRPC --> S1 + S1 --> R1 + S2 --> R1 + S2 --> R2 + S3 --> R1 + S3 --> R2 + R1 --> CH + R2 --> KAFKA + R3 --> IAM + R3 --> R4 + R4 --> REDIS + C1 --> C2 + C1 --> R1 + C1 --> R4 + C1 --> KAFKA +``` ## 技术栈 -- Python 3.12+ / FastAPI 0.115+ -- clickhouse-connect（ClickHouse 宽表查询） -- pydantic + pydantic-settings（运行时校验与配置） -- structlog（结构化日志） -- prometheus-client（指标） -- OpenTelemetry（分布式追踪） - -## 开发 - -```bash -uv sync -uv run uvicorn src.data_ana.main:app --host 0.0.0.0 --port 3006 --reload -``` - -## 配置 - -| 变量 | 默认值 | 说明 | -|------|--------|------| -| `PORT` | 3006 | 服务端口 | -| `CLICKHOUSE_HOST` | localhost | ClickHouse 主机 | -| `CLICKHOUSE_PORT` | 8123 | ClickHouse HTTP 端口 | -| `CLICKHOUSE_DATABASE` | edu_analytics | ClickHouse 数据库 | -| `OTEL_ENDPOINT` | http://localhost:4318 | OpenTelemetry 端点 | -| `LOG_LEVEL` | info | 日志级别 | +- **Python 3.12+** / **FastAPI 0.115+**（HTTP 端点） +- **grpcio 1.66+** / **grpcio-health-checking**（gRPC server + HealthService） +- **clickhouse-connect 0.7+**（ClickHouse 宽表查询，支持 FINAL/argMax） +- **aiokafka 0.11+**（CDC 消费者，手动 commit + 幂等去重） +- **redis 5.0+**（DataScope 缓存 + 事件去重 + 预警位图） +- **pydantic 2.9+** / **pydantic-settings 2.5+**（模型校验 + 配置） +- **structlog 24.4+**（结构化 JSON 日志） +- **prometheus-client 0.20+**（/metrics 指标） +- **opentelemetry-sdk 1.27+**（OTLP 链路追踪） +- **protobuf 5.28+**（buf generate 生成的 stub 运行时） ## 模块结构 ``` src/data_ana/ ├─ __init__.py -├─ main.py # FastAPI 入口（健康检查 + 分析端点骨架） -├─ config.py # pydantic-settings 配置 -└─ clickhouse_client.py # ClickHouse 客户端单例 +├─ main.py # FastAPI 入口（14 端点 + lifespan + ActionState） +├─ config.py # pydantic-settings 配置（env_prefix=DATA_ANA_） +├─ analytics_service.py # 4 Dashboard + 班级/趋势查询编排 +├─ mastery_service.py # 掌握度算法（WMA + ForgettingCurve） +├─ warning_service.py # 5 类预警评估 + Redis 去重 +├─ exam_cache.py # LRU exam_id→metadata 映射（max 10000） +├─ cdc_consumer.py # CDC 消费者（7 类事件 + 手动 commit） +├─ grpc_server.py # 12 RPC + HealthService + SubscribeMasteryUpdate +├─ clickhouse_client.py # ClickHouse 客户端单例（降级模式） +├─ repository/ +│ ├─ __init__.py +│ ├─ clickhouse_repository.py # 宽表查询 + FINAL/argMax + DataScope WHERE +│ ├─ redis_client.py # Redis 异步客户端（cache + dedup + bitmap） +│ ├─ iam_client.py # iam gRPC GetEffectiveDataScope + 角色降级 +│ └─ kafka_producer.py # mastery.updated / warning.triggered 发布 +└─ shared/ + ├─ __init__.py + ├─ action_state.py # ActionState[T] 统一信封（ok/fail + degraded） + ├─ errors.py # ErrorCode 13 类 + DataAnaError + └─ permissions.py # 权限点 + DataScopeLevel + build_datascope_where ``` -## 关键端点 +## 开发 -- `GET /healthz` 健康检查 -- `GET /metrics` Prometheus 指标 -- `GET /analytics/class/{class_id}/performance` 班级成绩分析（P4 骨架） -- `GET /analytics/student/{student_id}/weakness` 学生薄弱知识点分析（P4 骨架） +```bash +# 安装依赖 +uv sync + +# 本地开发（HTTP + gRPC + CDC 同时启动） +uv run uvicorn src.data_ana.main:app --host 0.0.0.0 --port 3006 --reload + +# ruff 校验 +ruff check src/ + +# 类型检查（如已配置 mypy） +mypy src/ +``` + +## 配置项 + +所有配置使用 `DATA_ANA_` 前缀（pydantic-settings env_prefix）。 + +### 基础配置 + +| 变量 | 默认值 | 说明 | +| ------------------------ | --------------------- | ------------------------------------ | +| `DATA_ANA_HTTP_PORT` | 3006 | HTTP 端口（Gateway 直连降级） | +| `DATA_ANA_GRPC_PORT` | 50055 | gRPC 端口（AnalyticsService 主入口） | +| `DATA_ANA_DEV_MODE` | true | 开发模式（关闭签名校验） | +| `DATA_ANA_LOG_LEVEL` | info | 日志级别（debug/info/warning/error） | +| `DATA_ANA_OTEL_ENDPOINT` | http://localhost:4318 | OpenTelemetry OTLP 端点 | + +### 外部依赖（全部可选，未配置则降级模式） + +| 变量 | 默认值 | 说明 | +| ------------------------------ | ------------- | ------------------------------------------ | +| `DATA_ANA_CLICKHOUSE_HOST` | "" | ClickHouse 主机（空字符串=降级模式） | +| `DATA_ANA_CLICKHOUSE_PORT` | 8123 | ClickHouse HTTP 端口 | +| `DATA_ANA_CLICKHOUSE_DATABASE` | edu_analytics | ClickHouse 数据库 | +| `DATA_ANA_CLICKHOUSE_USER` | "" | ClickHouse 用户名 | +| `DATA_ANA_CLICKHOUSE_PASSWORD` | "" | ClickHouse 密码 | +| `DATA_ANA_KAFKA_BROKERS` | "" | Kafka broker 列表（逗号分隔，空=禁用 CDC） | +| `DATA_ANA_KAFKA_GROUP_ID` | data-ana-cdc | 消费者组 ID | +| `DATA_ANA_REDIS_URL` | "" | Redis URL（空=禁用缓存/去重） | +| `DATA_ANA_IAM_GRPC_ENDPOINT` | "" | iam gRPC 端点（空=角色降级） | + +### 降级模式说明 + +- **ClickHouse 未配置/不可达**：查询返回骨架数据（空数组 + degraded=true） +- **Kafka 未配置**：CDC 消费者不启动，`/readyz` 仍就绪 +- **Redis 未配置/不可达**：缓存失效，每次查询重新计算 DataScope +- **iam gRPC 未配置/不可达**：基于 JWT 角色 fallback（teacher→CLASS，student→SELF） +- **grpcio 未安装**：gRPC server 不启动，HTTP 仍可用 + +## 端点清单 + +### 基础端点（3 个） + +| 方法 | 路径 | 说明 | +| ---- | ---------- | ----------------------------------------------------------- | +| GET | `/` | 根信息（服务名/版本/端点列表） | +| GET | `/healthz` | liveness 检查（进程存活即 ok） | +| GET | `/readyz` | readiness 检查（5 依赖状态：clickhouse/cdc/redis/iam/grpc） | +| GET | `/metrics` | Prometheus 指标 | + +### 业务端点（11 个，全部返回 ActionState[T]） + +| 方法 | 路径 | 说明 | +| ---- | -------------------------------------------------- | ------------------------------- | +| GET | `/analytics/class/{class_id}/performance` | 班级成绩分析（平均分/及格率） | +| GET | `/analytics/class/{class_id}/mastery-distribution` | 班级掌握度分布（三档） | +| GET | `/analytics/student/{student_id}/weakness` | 学生薄弱知识点（mastery < 0.6） | +| GET | `/analytics/student/{student_id}/mastery` | 学生掌握度明细 | +| GET | `/analytics/student/{student_id}/trend` | 学习趋势（历史成绩曲线） | +| GET | `/analytics/student/{student_id}/errorbook` | 学生错题本 | +| GET | `/analytics/teacher/dashboard` | 教师仪表盘 | +| GET | `/analytics/student/dashboard` | 学生仪表盘 | +| GET | `/analytics/parent/dashboard` | 家长仪表盘 | +| GET | `/analytics/admin/dashboard` | 管理员仪表盘 | +| GET | `/analytics/warnings` | 预警列表 | +| POST | `/analytics/warnings/trigger` | 手动触发预警 | + +### ActionState 信封格式 + +```json +{ + "success": true, + "data": { ... }, + "details": { + "degraded": false, + "degraded_reason": "" + } +} +``` + +降级时 `degraded=true`，`data` 返回骨架数据（空数组/零值）。 + +## gRPC 契约 + +### AnalyticsService（12 RPC，端口 50055） + +| RPC | 类型 | 说明 | +| ---------------------- | ---------------- | ----------------------------------- | +| GetClassPerformance | unary | 班级成绩分析 | +| GetStudentWeakness | unary | 学生薄弱知识点 | +| GetLearningTrend | unary | 学习趋势 | +| GetTeacherDashboard | unary | 教师仪表盘 | +| GetStudentDashboard | unary | 学生仪表盘 | +| GetParentDashboard | unary | 家长仪表盘 | +| GetAdminDashboard | unary | 管理员仪表盘 | +| GetWarnings | unary | 预警列表 | +| TriggerWarning | unary | 手动触发预警 | +| GetMasteryDistribution | unary | 班级掌握度分布 | +| GetStudentMastery | unary | 学生掌握度明细 | +| SubscribeMasteryUpdate | server-streaming | 掌握度更新订阅（P5+ AI 个性化通道） | + +### HealthService + +`grpc.health.v1.Health` 标准 K8s 探针。 + +### proto 定义 + +- `packages/shared-proto/proto/analytics.proto`（12 RPC + 全部消息） +- `packages/shared-proto/proto/events.proto`（MasteryEvent + AIUsageEvent） +- `packages/shared-proto/proto/iam.proto`（GetEffectiveDataScope） + +## CDC 事件消费 + +### 消费的 7 类事件 + +| 事件源 | Topic | 处理 | +| ------------------------ | ------------------------------------------------- | ------------------------- | +| core-edu.exams | `edu-cdc.next_edu_cloud.core_edu_exams` | 写 exam_cache | +| core-edu.grades | `edu-cdc.next_edu_cloud.core_edu_grades` | 写 student_dashboard_view | +| core-edu.homework | `edu-cdc.next_edu_cloud.core_edu_homework` | 写 student_dashboard_view | +| core-edu.attendance | `edu-cdc.next_edu_cloud.core_edu_attendance` | 写 attendance_logs | +| content.knowledge_points | `edu-cdc.next_edu_cloud.content_knowledge_points` | 知识点元数据 | +| ai.ai_usage | `edu.insight.ai.usage` | 写 ai_usage_log | + +### 幂等保证 + +- **Kafka producer**：`enable_idempotence=True` + `transactional_id` +- **Consumer 去重**：Redis SETNX `data_ana:dedup:{event_id}` TTL 7 天 +- **ClickHouse 去重**：ReplacingMergeTree(last_updated) + FINAL/argMax 查询 +- **手动 commit**：仅在 ClickHouse 写入成功后 commit（at-least-once 语义） + +## 发布的派生数据事件 + +| 事件 | Topic | 触发条件 | Outbox 豁免 | +| ---------------- | ------------------------------------- | -------------- | ----------------------- | +| MasteryUpdated | `edu.insight.mastery.updated` | 掌握度计算完成 | ✅（004 §12.2 仲裁） | +| WarningTriggered | `edu.insight.mastery.updated`（复用） | 预警阈值触发 | ✅（004 §15.3 #6 仲裁） | + +> Outbox 豁免理由：派生数据事件，无业务状态变更，重复消费幂等。 + +## ClickHouse 宽表 + +DDL 位置：`scripts/clickhouse-init.sql` + +| 表名 | 引擎 | ORDER BY | 用途 | +| ---------------------- | ----------------------------------- | ----------------------------------------------- | -------------- | +| student_dashboard_view | ReplacingMergeTree(last_updated) | (student_id, exam_id, knowledge_point_id) | 学情宽表 | +| student_errors | ReplacingMergeTree(last_error_time) | (student_id, question_id) | 错题本 | +| mastery_snapshot | MergeTree | (student_id, knowledge_point_id, calculated_at) | 掌握度历史快照 | +| ai_usage_log | ReplacingMergeTree(occurred_at) | (request_id,) | AI 用量计费 | +| attendance_logs | ReplacingMergeTree(occurred_at) | (student_id, class_id, attendance_date) | 考勤记录 | + +Mock 数据：`scripts/data-ana-mock-data.sql` + +## 掌握度算法 + +### 公式 + +``` +mastery = max(WMA, ForgettingCurve) + +WMA = Σ(w_i * score_i) / Σ(w_i), w_i = 0.6^i （近 5 次，权重递减） + +ForgettingCurve = last_mastery * exp(-ln(2) * days_since / 30) +``` + +### 分类 + +| 区间 | label | 说明 | +| --------- | ----------- | ------ | +| ≥ 0.8 | mastered | 已掌握 | +| 0.4 - 0.8 | progressing | 进展中 | +| < 0.4 | weak | 薄弱 | + +### 降级规则 + +- 近 5 次成绩不足 5 次 → 降级 SIMPLE_AVG +- 无历史数据 → 返回 0.0 + label=weak + +## 预警类型 + +| 类型 | 阈值 | severity | 说明 | +| --------------- | ------------- | -------- | -------- | +| LOW_MASTERY | mastery < 0.4 | WARN | 掌握度低 | +| CRITICAL_LOW | mastery < 0.2 | CRITICAL | 严重低 | +| SCORE_DROP | 下降 ≥ 20% | WARN | 成绩下降 | +| ABSENT_FREQUENT | ≥ 3 次/周 | WARN | 频繁缺勤 | +| TREND_DECLINE | 连续 3 次降 | INFO | 趋势下降 | + +### 去重 + +- Redis key：`data_ana:warning:{target_id}:{type}:{date}` +- TTL：25 小时（跨日保证） + +## DataScope 过滤 + +### 6 级 + +| 级别 | scope_ids 含义 | +| -------- | --------------------- | +| SELF | 学生本人 ID | +| CLASS | 可见 class_id 列表 | +| GRADE | 可见 grade_id 列表 | +| SCHOOL | 可见 school_id 列表 | +| DISTRICT | 可见 district_id 列表 | +| ALL | 无过滤 | + +### 解析链 + +``` +请求 → Redis 缓存（5min）→ iam gRPC GetEffectiveDataScope → 角色降级 fallback +``` + +### 角色降级（iam 不可用时） + +| JWT 角色 | 降级 scope | +| -------- | --------------------- | +| teacher | CLASS（自己教的班级） | +| student | SELF（本人） | +| parent | SELF（本人 + 孩子） | +| admin | ALL | + +## 部署 + +### Docker + +```bash +# 构建（多阶段：builder + runtime） +docker build -t data-ana:1.0.0 services/data-ana/ + +# 运行（需 ClickHouse/Kafka/Redis 可达） +docker run -p 3006:3006 -p 50055:50055 \ + -e DATA_ANA_CLICKHOUSE_HOST=clickhouse \ + -e DATA_ANA_KAFKA_BROKERS=kafka:9092 \ + -e DATA_ANA_REDIS_URL=redis://redis:6379 \ + data-ana:1.0.0 +``` + +### 健康检查 + +- HTTP `/healthz`：进程存活即 ok +- HTTP `/readyz`：检查 5 依赖（clickhouse/cdc/redis/iam/grpc） +- gRPC `grpc.health.v1.Health`：K8s 探针 ## 对外契约 -gRPC 服务 `AnalyticsService` 定义见 `packages/shared-proto/proto/analytics.proto`。 +- **gRPC proto**：`packages/shared-proto/proto/analytics.proto` +- **HTTP OpenAPI**：启动后访问 `/docs` +- **CDC topic 命名**：`edu-cdc.next_edu_cloud.`（Debezium 标准） +- **派生事件 topic**：`edu.insight.mastery.updated` / `edu.insight.ai.usage` + +## 相关文档 + +- [02-architecture-design.md](./docs/02-architecture-design.md)：模块架构设计 v2（仲裁后） +- [01-understanding.md](./docs/01-understanding.md)：阶段 1 理解确认书 +- [004 架构影响地图 §15.3](../../docs/architecture/004_architecture_impact_map.md)：coord 仲裁结论 +- [coord-cross-review.md](../../docs/architecture/coord-cross-review.md)：交叉审查记录 +- [known-issues §2.6](../../docs/troubleshooting/known-issues.md)：data-ana 已知问题速查 diff --git a/services/data-ana/pyproject.toml b/services/data-ana/pyproject.toml index 64d7db8..546e250 100644 --- a/services/data-ana/pyproject.toml +++ b/services/data-ana/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "data-ana-service" -version = "0.1.0" -description = "数据分析服务 - ClickHouse + 学习分析" +version = "1.0.0" +description = "数据分析服务 - ClickHouse 宽表 + CDC + 掌握度算法 + 预警" requires-python = ">=3.12" dependencies = [ "fastapi>=0.115.0", @@ -17,6 +17,13 @@ dependencies = [ "structlog>=24.4.0", # CDC 链路：消费 Debezium 写入 Kafka 的 MySQL binlog 变更事件 "aiokafka>=0.11.0", + # Redis：DataScope 缓存 + 事件去重 + 预警去重位图 + "redis>=5.0.0", + # gRPC：AnalyticsService 12 RPC + HealthService（端口 50055） + "grpcio>=1.66.0", + "grpcio-health-checking>=1.66.0", + # protobuf 运行时（buf generate 生成的 stub 依赖） + "protobuf>=5.28.0", ] [tool.ruff] @@ -25,3 +32,9 @@ target-version = "py312" [tool.ruff.lint] select = ["E", "F", "I", "N", "W", "UP", "B", "SIM"] + +[tool.ruff.lint.per-file-ignores] +# gRPC servicer 方法必须用 PascalCase（proto 契约约定），N802 不适用 +"src/data_ana/grpc_server.py" = ["N802"] +# FastAPI 标准模式：Depends/Query 在参数默认值中调用，B008 不适用 +"src/data_ana/main.py" = ["B008"] diff --git a/services/data-ana/src/data_ana/analytics_service.py b/services/data-ana/src/data_ana/analytics_service.py new file mode 100644 index 0000000..48b6729 --- /dev/null +++ b/services/data-ana/src/data_ana/analytics_service.py @@ -0,0 +1,450 @@ +"""分析服务（4 端 Dashboard 聚合 + DataScope 注入 + 降级兜底）. + +对齐 02-architecture-design.md §7 AnalyticsService： + - GetTeacherDashboard：班级聚合（教师视角） + - GetStudentDashboard：学生学情（学生视角） + - GetParentDashboard：孩子学情（家长视角） + - GetAdminDashboard：学校聚合 + AI 用量（管理员视角） + +DataScope 注入： + - 教师只能查自己班级数据（CLASS 级 scope_ids 过滤） + - 学生只能查自己数据（SELF 级 user_id 过滤） + - 管理员可查全校数据（ALL 级无过滤） + +降级策略： + - ClickHouse 不可达：返回骨架数据 + degraded=true + - iam gRPC 不可达：使用 role-based fallback + degraded=true +""" + +from typing import Any + +import structlog + +from .repository import clickhouse_repository, iam_client +from .shared.permissions import DataScopeLevel, UserContext + +logger = structlog.get_logger(__name__) + + +def _skeleton_dashboard(dashboard_type: str, user_id: str, reason: str) -> dict[str, Any]: + """降级骨架数据（ClickHouse 不可达时返回）.""" + return { + "userId": user_id, + "dashboardType": dashboard_type, + "degraded": True, + "degraded_reason": reason, + "totalClasses": 0, + "totalStudents": 0, + "averageScore": 0.0, + "passRate": 0.0, + "weakStudents": [], + "recentWarnings": [], + "trend": [], + } + + +async def get_teacher_dashboard( + user: UserContext, + class_id: str = "", + subject_id: str = "", +) -> dict[str, Any]: + """教师仪表盘（聚合班级成绩 + 薄弱学生 + 预警）. + + DataScope： + - CLASS 级：仅返回 scope_ids 内的班级数据 + - ALL 级：返回所有班级数据 + """ + # 1. 解析 DataScope + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # 2. 若指定 class_id，验证是否在 scope 内 + effective_class_ids: list[str] = [] + if class_id: + if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids: + return { + **_skeleton_dashboard("teacher", user.user_id, "datascope_violation"), + "error": "class_id_out_of_scope", + } + effective_class_ids = [class_id] + elif scope.level == DataScopeLevel.CLASS: + effective_class_ids = scope.scope_ids + + # 3. 查询 ClickHouse + if not effective_class_ids and scope.level != DataScopeLevel.ALL: + # CLASS 级无 scope_ids，返回空骨架 + return _skeleton_dashboard("teacher", user.user_id, "no_class_scope") + + dashboard = await clickhouse_repository.query_teacher_dashboard( + user_id=user.user_id, + class_id=effective_class_ids[0] if effective_class_ids else "", + ) + if dashboard is None: + return _skeleton_dashboard("teacher", user.user_id, "clickhouse_unavailable") + + # 4. 补充薄弱学生列表 + weak_students: list[dict[str, Any]] = [] + if effective_class_ids: + for cid in effective_class_ids[:5]: # 限制 5 个班级 + class_perf = await clickhouse_repository.query_class_performance( + class_id=cid, + subject_id=subject_id, + ) + if class_perf is not None: + weak_students.append( + { + "class_id": cid, + "total_students": class_perf.get("totalStudents", 0), + "average_score": class_perf.get("averageScore", 0.0), + "pass_rate": class_perf.get("passRate", 0.0), + } + ) + + # 5. 补充降级标记 + if degraded: + dashboard["degraded"] = True + dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + + dashboard["classes"] = weak_students + dashboard["dashboardType"] = "teacher" + return dashboard + + +async def get_student_dashboard( + user: UserContext, + student_id: str = "", + subject_id: str = "", +) -> dict[str, Any]: + """学生仪表盘（学情宽表 + 薄弱知识点 + 学习趋势）. + + DataScope： + - SELF 级：student_id 必须为当前 user_id + - CLASS/GRADE/SCHOOL/ALL 级：可查指定 student_id + """ + # 1. 解析 DataScope + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # 2. SELF 级强制限制 student_id + effective_student_id = student_id or user.user_id + if scope.level == DataScopeLevel.SELF: + effective_student_id = user.user_id # SELF 级只能查自己 + + # 3. 查询 ClickHouse + dashboard = await clickhouse_repository.query_student_dashboard(effective_student_id) + if dashboard is None: + return _skeleton_dashboard("student", effective_student_id, "clickhouse_unavailable") + + # 4. 补充薄弱知识点 + weakness = await clickhouse_repository.query_student_weakness( + student_id=effective_student_id, + subject_id=subject_id, + ) + dashboard["weakPoints"] = weakness.get("weakPoints", []) if weakness else [] + + # 5. 补充学习趋势 + trend = await clickhouse_repository.query_learning_trend( + student_id=effective_student_id, + subject_id=subject_id, + ) + dashboard["trend"] = trend.get("points", []) if trend else [] + + # 6. 补充掌握度快照 + mastery = await clickhouse_repository.query_mastery_snapshot( + student_id=effective_student_id, + subject_id=subject_id, + ) + dashboard["mastery"] = mastery if mastery else {"knowledgePoints": [], "overallMastery": 0.0} + + # 7. 降级标记 + if degraded: + dashboard["degraded"] = True + dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + + dashboard["dashboardType"] = "student" + return dashboard + + +async def get_parent_dashboard( + user: UserContext, + child_id: str = "", +) -> dict[str, Any]: + """家长仪表盘（孩子学情聚合）. + + DataScope： + - SELF 级：child_id 必须为家长关联的孩子（iam.GetChildrenByParent 校验） + - 降级时使用 child_id 直接查询（依赖 Gateway 层鉴权） + """ + # 1. 解析 DataScope + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # 2. 学生 ID（家长视角：使用 child_id 或 user_id） + effective_student_id = child_id or user.user_id + + # 3. 查询 ClickHouse（复用学生仪表盘查询） + dashboard = await clickhouse_repository.query_student_dashboard(effective_student_id) + if dashboard is None: + return _skeleton_dashboard("parent", effective_student_id, "clickhouse_unavailable") + + # 4. 补充考勤 + attendance = await clickhouse_repository.query_attendance(effective_student_id) + dashboard["attendance"] = ( + attendance + if attendance + else { + "absentCount": 0, + "lateCount": 0, + "presentCount": 0, + } + ) + + # 5. 补充学习趋势 + trend = await clickhouse_repository.query_learning_trend(effective_student_id) + dashboard["trend"] = trend.get("points", []) if trend else [] + + # 6. 降级标记 + if degraded: + dashboard["degraded"] = True + dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + + dashboard["dashboardType"] = "parent" + dashboard["childId"] = effective_student_id + return dashboard + + +async def get_admin_dashboard( + user: UserContext, + school_id: str = "", +) -> dict[str, Any]: + """管理员仪表盘（学校聚合 + AI 用量统计）. + + DataScope： + - SCHOOL 级：仅返回指定 school_id 数据 + - ALL 级：返回所有数据 + """ + # 1. 解析 DataScope + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # 2. 学校级聚合查询（复用 teacher_dashboard，不传 class_id） + dashboard = await clickhouse_repository.query_teacher_dashboard( + user_id=user.user_id, + class_id="", + ) + if dashboard is None: + return _skeleton_dashboard("admin", user.user_id, "clickhouse_unavailable") + + # 3. 补充 AI 用量统计（P5 启用，对齐 02 §7.4） + ai_usage = await clickhouse_repository.query_ai_usage_summary() + dashboard["aiUsage"] = ( + ai_usage + if ai_usage + else { + "totalRequests": 0, + "totalTokens": 0, + "totalCostCents": 0, + "byProvider": [], + } + ) + + # 4. 降级标记 + if degraded: + dashboard["degraded"] = True + dashboard["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + + dashboard["dashboardType"] = "admin" + dashboard["schoolId"] = school_id or scope.school_id + return dashboard + + +async def get_class_performance( + user: UserContext, + class_id: str, + subject_id: str = "", + start_date: int = 0, + end_date: int = 0, +) -> dict[str, Any]: + """查询班级成绩分析（GetClassPerformance RPC 实现）. + + DataScope：CLASS 级需验证 class_id 在 scope_ids 内. + """ + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # DataScope 校验 + if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids: + return { + "classId": class_id, + "degraded": True, + "degraded_reason": "datascope_violation", + "averageScore": 0.0, + "passRate": 0.0, + "totalStudents": 0, + } + + result = await clickhouse_repository.query_class_performance( + class_id=class_id, + subject_id=subject_id, + start_date=start_date, + end_date=end_date, + ) + if result is None: + return { + "classId": class_id, + "degraded": True, + "degraded_reason": "clickhouse_unavailable", + "averageScore": 0.0, + "passRate": 0.0, + "totalStudents": 0, + } + + if degraded: + result["degraded"] = True + result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + return result + + +async def get_mastery_distribution( + user: UserContext, + class_id: str, + subject_id: str = "", + knowledge_point_id: str = "", +) -> dict[str, Any]: + """查询班级掌握度分布（GetMasteryDistribution RPC 实现）.""" + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # DataScope 校验 + if scope.level == DataScopeLevel.CLASS and class_id not in scope.scope_ids: + return { + "classId": class_id, + "degraded": True, + "degraded_reason": "datascope_violation", + "masteredCount": 0, + "progressingCount": 0, + "weakCount": 0, + "totalStudents": 0, + } + + result = await clickhouse_repository.query_mastery_distribution( + class_id=class_id, + subject_id=subject_id, + knowledge_point_id=knowledge_point_id, + ) + if result is None: + return { + "classId": class_id, + "degraded": True, + "degraded_reason": "clickhouse_unavailable", + "masteredCount": 0, + "progressingCount": 0, + "weakCount": 0, + "totalStudents": 0, + } + + if degraded: + result["degraded"] = True + result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + return result + + +async def get_student_mastery( + user: UserContext, + student_id: str, + subject_id: str = "", +) -> dict[str, Any]: + """查询学生知识点掌握度明细（GetStudentMastery RPC 实现）.""" + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # SELF 级强制限制 + effective_student_id = student_id + if scope.level == DataScopeLevel.SELF: + effective_student_id = user.user_id + + result = await clickhouse_repository.query_mastery_snapshot( + student_id=effective_student_id, + subject_id=subject_id, + ) + if result is None: + return { + "studentId": effective_student_id, + "degraded": True, + "degraded_reason": "clickhouse_unavailable", + "knowledgePoints": [], + "overallMastery": 0.0, + } + + if degraded: + result["degraded"] = True + result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + return result + + +async def get_student_weakness( + user: UserContext, + student_id: str, + subject_id: str = "", +) -> dict[str, Any]: + """查询学生薄弱知识点（GetStudentWeakness RPC 实现）.""" + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # SELF 级强制限制 + effective_student_id = student_id + if scope.level == DataScopeLevel.SELF: + effective_student_id = user.user_id + + result = await clickhouse_repository.query_student_weakness( + student_id=effective_student_id, + subject_id=subject_id, + ) + if result is None: + return { + "studentId": effective_student_id, + "degraded": True, + "degraded_reason": "clickhouse_unavailable", + "weakPoints": [], + } + + if degraded: + result["degraded"] = True + result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + return result + + +async def get_learning_trend( + user: UserContext, + student_id: str, + subject_id: str = "", + start_date: int = 0, + end_date: int = 0, +) -> dict[str, Any]: + """查询学习趋势（GetLearningTrend RPC 实现）.""" + scope = await iam_client.get_effective_datascope(user) + degraded = scope.degraded + + # SELF 级强制限制 + effective_student_id = student_id + if scope.level == DataScopeLevel.SELF: + effective_student_id = user.user_id + + result = await clickhouse_repository.query_learning_trend( + student_id=effective_student_id, + subject_id=subject_id, + start_date=start_date, + end_date=end_date, + ) + if result is None: + return { + "studentId": effective_student_id, + "degraded": True, + "degraded_reason": "clickhouse_unavailable", + "points": [], + } + + if degraded: + result["degraded"] = True + result["degraded_reason"] = scope.degraded_reason or "iam_grpc_unavailable" + return result diff --git a/services/data-ana/src/data_ana/cdc_consumer.py b/services/data-ana/src/data_ana/cdc_consumer.py index d7f7d8d..36511e4 100644 --- a/services/data-ana/src/data_ana/cdc_consumer.py +++ b/services/data-ana/src/data_ana/cdc_consumer.py @@ -1,17 +1,28 @@ -"""CDC 消费者（Debezium MySQL binlog → ClickHouse 宽表）. +"""CDC 消费者（Debezium MySQL binlog → ClickHouse 宽表，EventHandler 路由 + 手动 commit）. 链路： MySQL binlog → Debezium Connect → Kafka topic edu-cdc.next_edu_cloud.
- → 本消费者 → 解析 Debezium 事件 → 写入 ClickHouse student_dashboard_view + → 本消费者 → 解析 Debezium 事件 → EventHandler 路由 → 写入 ClickHouse 宽表 + +支持的表路由： + - core_edu_exams → ExamCache 更新 + - core_edu_grades → student_dashboard_view 写入（查 ExamCache 获取 class_id） + - core_edu_homework_submissions → student_dashboard_view 写入 + - core_edu_attendance → attendance_logs 写入 + - content_knowledge_points → mastery_snapshot 元数据更新 + - iam_users → 用户元数据缓存（P6 扩展） + - classes → 班级元数据缓存（P6 扩展） 设计要点： -- 监听多张表，按 source.table 路由 -- 内存缓存 exam_id → (class_id, subject_id) 映射（来自 core_edu_exams 快照+流） -- 监听 core_edu_grades 时用缓存扩展为宽表记录写入 ClickHouse -- 幂等性：依赖 ClickHouse ReplacingMergeTree 引擎按 ORDER BY 去重 - （schema 需用 ReplacingMergeTree(last_updated)，当前为简化版 MergeTree） -- op 类型：r(快照读)、c(新增)、u(更新)、d(删除)；d 时 after 为 null + - 手动 commit（at-least-once）：ClickHouse 写入成功后才 commit offset + - 幂等性：依赖 ClickHouse ReplacingMergeTree 引擎按 ORDER BY 去重 + - op 类型：r(快照读)、c(新增)、u(更新)、d(删除)；d 时 after 为 null + - Redis 事件去重：基于 event_id（Debezium 重启时可能重发） + +降级策略： + - kafka_brokers 未配置：消费者不启动（仅 HTTP/gRPC 服务） + - ClickHouse 不可达：消息处理失败，不 commit，下次重启重试 """ import asyncio @@ -22,11 +33,17 @@ from typing import Any import structlog -from .clickhouse_client import upsert_student_dashboard from .config import settings +from .exam_cache import get_exam_cache +from .repository import clickhouse_repository, redis_client logger = structlog.get_logger(__name__) +# 消费者单例（用于 lifespan 管理和 /readyz 检查） +_consumer: Any | None = None +_consumer_task: asyncio.Task[None] | None = None +_is_running: bool = False + def _parse_ts(ts_ms: int | None) -> datetime: """Debezium ts_ms（毫秒）→ datetime.""" @@ -45,103 +62,291 @@ def _safe_float(value: Any) -> float: return 0.0 -class ExamCache: - """内存缓存 exam_id → (class_id, subject_id). - - 从 core_edu_exams 表的 CDC 事件构建。subject_id 在 exams 表中暂无字段， - 这里占位为空字符串，后续扩展 schema 时再补充。 - """ - - def __init__(self) -> None: - self._data: dict[str, tuple[str, str]] = {} - - def upsert(self, exam_id: str, class_id: str, subject_id: str = "") -> None: - if exam_id: - self._data[exam_id] = (class_id, subject_id) - - def get(self, exam_id: str) -> tuple[str, str] | None: - return self._data.get(exam_id) if exam_id else None +def _safe_int(value: Any) -> int: + """安全转 int.""" + if value is None: + return 0 + try: + return int(value) + except (TypeError, ValueError): + try: + return int(float(value)) + except (TypeError, ValueError): + return 0 -# 全局缓存（进程级单例） -_exam_cache = ExamCache() +def _parse_mysql_datetime(value: Any, fallback: datetime) -> datetime: + """解析 MySQL datetime 字段（兼容字符串和毫秒时间戳）.""" + if not value: + return fallback + if isinstance(value, (int, float)): + return datetime.fromtimestamp(value / 1000, tz=UTC) + with contextlib.suppress(ValueError, AttributeError, TypeError): + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + return fallback -async def _handle_exams_event(after: dict[str, Any] | None) -> None: - """处理 core_edu_exams 表事件.""" +# ===== EventHandler 路由表 ===== + + +async def _handle_exams_event(after: dict[str, Any] | None, op: str) -> bool: + """处理 core_edu_exams 表事件 → ExamCache 更新.""" if after is None: - return - exam_id = after.get("id") - class_id = after.get("class_id", "") - if exam_id: - _exam_cache.upsert(exam_id, class_id) - logger.info("exam_cache_updated", exam_id=exam_id, class_id=class_id) + return True # 删除事件，无需处理 + + exam_id = str(after.get("id") or "") + if not exam_id: + return True + + exam_cache = get_exam_cache() + exam_cache.upsert( + exam_id=exam_id, + class_id=str(after.get("class_id") or ""), + subject_id=str(after.get("subject_id") or ""), + title=str(after.get("title") or ""), + ) + logger.info( + "exam_cache_updated", + exam_id=exam_id, + class_id=after.get("class_id"), + op=op, + ) + return True async def _handle_grades_event( after: dict[str, Any] | None, op: str, ts_ms: int | None, -) -> None: - """处理 core_edu_grades 表事件 → 写入 ClickHouse 宽表. - - - op=r/c/u：after 为新数据，写入宽表 - - op=d：after 为 null，暂不处理（宽表保留历史记录） - """ +) -> bool: + """处理 core_edu_grades 表事件 → 写入 student_dashboard_view.""" if after is None: - return + return True # 删除事件 - student_id = after.get("student_id", "") - exam_id = after.get("exam_id", "") + student_id = str(after.get("student_id") or "") + exam_id = str(after.get("exam_id") or "") score = _safe_float(after.get("score")) - # 从缓存拿 class_id - class_id = "" - if exam_id: - cached = _exam_cache.get(exam_id) - if cached: - class_id = cached[0] + # 从 ExamCache 获取 class_id 和 subject_id + exam_cache = get_exam_cache() + class_id = exam_cache.get_class_id(exam_id) + subject_id = exam_cache.get_subject_id(exam_id) - last_updated = _parse_ts(ts_ms) - if after.get("updated_at"): - # 优先用 MySQL 的 updated_at 字段 - with contextlib.suppress(ValueError, AttributeError): - last_updated = datetime.fromisoformat(after["updated_at"].replace("Z", "+00:00")) + fallback_ts = _parse_ts(ts_ms) + last_updated = _parse_mysql_datetime(after.get("updated_at"), fallback_ts) - # 简化：rank/kp/mastery/error_count 暂用默认值 - # 真实场景应通过其他 CDC 事件或聚合计算得到 - await upsert_student_dashboard( + # 简化：rank/kp/mastery/error_count 暂用默认值（CDC 聚合后由掌握度计算补充） + write_ok = await clickhouse_repository.upsert_student_dashboard( student_id=student_id, class_id=class_id, exam_id=exam_id, - subject_id="", # 占位 + subject_id=subject_id, score=score, rank_in_class=0, - knowledge_point_id="", # 占位 - mastery_level=score / 100.0, # 简化：用分数百分比作为掌握度 + knowledge_point_id="", + mastery_level=score / 100.0, # 简化：用分数百分比作为初始掌握度 error_count=0, last_updated=last_updated, ) + if write_ok: + logger.info( + "grade_record_written", + student_id=student_id, + exam_id=exam_id, + score=score, + op=op, + ) + return write_ok -async def _process_message(topic: str, value: bytes | str) -> None: - """处理单条 Kafka 消息. +async def _handle_homework_event( + after: dict[str, Any] | None, + op: str, + ts_ms: int | None, +) -> bool: + """处理 core_edu_homework_submissions 表事件 → 写入 student_dashboard_view.""" + if after is None: + return True - Debezium 事件格式（简化后，schemas.enable=false）： - { - "before": {...} | null, - "after": {...} | null, - "source": {"table": "...", "db": "...", ...}, - "op": "r|c|u|d", - "ts_ms": 1783572350928 - } + student_id = str(after.get("student_id") or "") + homework_id = str(after.get("homework_id") or after.get("id") or "") + class_id = str(after.get("class_id") or "") + score = _safe_float(after.get("score")) + + fallback_ts = _parse_ts(ts_ms) + raw_time = after.get("submitted_at") or after.get("updated_at") + last_updated = _parse_mysql_datetime(raw_time, fallback_ts) + + write_ok = await clickhouse_repository.upsert_student_dashboard( + student_id=student_id, + class_id=class_id, + exam_id=homework_id, # homework_id 作为 exam_id 占位 + subject_id=str(after.get("subject_id") or ""), + score=score, + rank_in_class=0, + knowledge_point_id="", + mastery_level=score / 100.0, + error_count=0, + last_updated=last_updated, + ) + if write_ok: + logger.info( + "homework_record_written", + student_id=student_id, + homework_id=homework_id, + score=score, + op=op, + ) + return write_ok + + +async def _handle_attendance_event( + after: dict[str, Any] | None, + op: str, + ts_ms: int | None, +) -> bool: + """处理 core_edu_attendance 表事件 → 写入 attendance_logs.""" + if after is None: + return True + + student_id = str(after.get("student_id") or "") + class_id = str(after.get("class_id") or "") + status = str(after.get("status") or "present") + recorded_by = str(after.get("recorded_by") or "") + remark = str(after.get("remark") or "") + + fallback_ts = _parse_ts(ts_ms) + raw_time = after.get("occurred_at") or after.get("created_at") + occurred_at = _parse_mysql_datetime(raw_time, fallback_ts) + + # attendance_date 从 occurred_at 提取日期部分 + attendance_date = occurred_at.date() + + write_ok = await clickhouse_repository.upsert_attendance_log( + student_id=student_id, + class_id=class_id, + attendance_date=attendance_date, + status=status, + recorded_by=recorded_by, + remark=remark, + occurred_at=occurred_at, + ) + if write_ok: + logger.info( + "attendance_record_written", + student_id=student_id, + class_id=class_id, + status=status, + op=op, + ) + return write_ok + + +async def _handle_knowledge_point_event( + after: dict[str, Any] | None, + op: str, +) -> bool: + """处理 content_knowledge_points 表事件 → 更新知识点元数据缓存.""" + if after is None: + return True + + # 知识点元数据更新（P5+ 扩展：可写入 Redis 缓存供查询时补充标题） + kp_id = str(after.get("id") or "") + title = str(after.get("title") or "") + subject_id = str(after.get("subject_id") or "") + + # 简化：写 Redis 缓存（data_ana:kp_meta:{kp_id}） + metadata = {"title": title, "subject_id": subject_id} + await redis_client.set_cache( + f"data_ana:kp_meta:{kp_id}", + json.dumps(metadata, ensure_ascii=False), + ttl_s=86400, + ) + logger.info( + "knowledge_point_meta_cached", + knowledge_point_id=kp_id, + title=title, + op=op, + ) + return True + + +async def _handle_ai_usage_event(event: dict[str, Any]) -> bool: + """处理 AIUsageEvent（edu.insight.ai.usage topic）→ 写入 ai_usage_log. + + 注意：此事件不是 Debezium CDC 事件，而是 ai 服务发布的业务事件， + 格式遵循 events.proto AIUsageEvent message. + """ + request_id = str(event.get("request_id") or "") + user_id = str(event.get("user_id") or "") + provider = str(event.get("provider") or "") + model = str(event.get("model") or "") + prompt_tokens = _safe_int(event.get("prompt_tokens")) + completion_tokens = _safe_int(event.get("completion_tokens")) + total_tokens = _safe_int(event.get("total_tokens")) + latency_ms = _safe_int(event.get("latency_ms")) + success = bool(event.get("success", True)) + cost_cents = _safe_int(event.get("cost_cents")) + occurred_at_ms = event.get("occurred_at") + if isinstance(occurred_at_ms, int): + occurred_at = _parse_ts(occurred_at_ms) + else: + occurred_at = datetime.now(UTC) + + write_ok = await clickhouse_repository.upsert_ai_usage_log( + request_id=request_id, + user_id=user_id, + provider=provider, + model=model, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + latency_ms=latency_ms, + success=success, + cost_cents=cost_cents, + occurred_at=occurred_at, + ) + if write_ok: + logger.info( + "ai_usage_recorded", + request_id=request_id, + provider=provider, + model=model, + total_tokens=total_tokens, + ) + return write_ok + + +# ===== 事件去重 ===== + + +async def _is_deduped(event_id: str) -> bool: + """检查事件是否已处理（基于 event_id Redis SETNX 去重）. + + 返回 True 表示重复事件（应跳过），False 表示首次事件. + """ + if not event_id: + return False + key = f"data_ana:dedup:{event_id}" + is_first = await redis_client.setnx_dedup(key, ttl_s=7 * 24 * 3600) # 7 天 TTL + return not is_first + + +# ===== 消息处理主入口 ===== + + +async def _process_cdc_message(topic: str, value: bytes | str) -> bool: + """处理单条 CDC 消息（Debezium 格式）. + + 返回 True 表示处理成功（可 commit offset）， + 返回 False 表示处理失败（不 commit，下次重试）. """ try: value_str = value.decode("utf-8") if isinstance(value, bytes) else value event = json.loads(value_str) except (json.JSONDecodeError, UnicodeDecodeError) as exc: logger.warning("cdc_message_decode_failed", error=str(exc), topic=topic) - return + return True # 无效消息直接跳过（避免阻塞消费） source = event.get("source") or {} table = source.get("table", "") @@ -149,6 +354,16 @@ async def _process_message(topic: str, value: bytes | str) -> None: ts_ms = event.get("ts_ms") after = event.get("after") + # Debezium 事件去重（基于 source.ts_ms + source.dbg + source.sequence 生成 event_id） + # 简化：使用 topic + partition + offset 作为去重 key（由调用方传入更精确） + # 这里仅对 after.id 做去重（如果有） + event_id = "" + if after and isinstance(after, dict): + event_id = f"{table}:{after.get('id', '')}:{ts_ms}" + if event_id and await _is_deduped(event_id): + logger.info("cdc_event_dedup_skipped", table=table, event_id=event_id) + return True + logger.info( "cdc_event_received", topic=topic, @@ -157,21 +372,71 @@ async def _process_message(topic: str, value: bytes | str) -> None: ts_ms=ts_ms, ) - if table == "core_edu_exams": - await _handle_exams_event(after) - elif table == "core_edu_grades": - await _handle_grades_event(after, op, ts_ms) - else: - # 其他表暂不处理，仅记录 - logger.debug("cdc_event_skipped", table=table) + try: + if table == "core_edu_exams": + return await _handle_exams_event(after, op) + elif table == "core_edu_grades": + return await _handle_grades_event(after, op, ts_ms) + elif table == "core_edu_homework_submissions": + return await _handle_homework_event(after, op, ts_ms) + elif table == "core_edu_attendance": + return await _handle_attendance_event(after, op, ts_ms) + elif table == "content_knowledge_points": + return await _handle_knowledge_point_event(after, op) + else: + logger.debug("cdc_event_skipped", table=table) + return True + except Exception as exc: # noqa: BLE001 + logger.error( + "cdc_event_handler_failed", + error=str(exc), + table=table, + op=op, + ) + return False + + +async def _process_ai_usage_message(topic: str, value: bytes | str) -> bool: + """处理 AIUsageEvent 消息（非 Debezium 格式，业务事件）.""" + try: + value_str = value.decode("utf-8") if isinstance(value, bytes) else value + event = json.loads(value_str) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + logger.warning("ai_usage_message_decode_failed", error=str(exc), topic=topic) + return True + + # 基于 request_id 去重 + request_id = str(event.get("request_id") or "") + if request_id and await _is_deduped(f"ai_usage:{request_id}"): + return True + + try: + return await _handle_ai_usage_event(event) + except Exception as exc: # noqa: BLE001 + logger.error("ai_usage_handler_failed", error=str(exc)) + return False + + +async def _process_message(topic: str, value: bytes | str) -> bool: + """统一消息处理入口（区分 CDC 事件和 AIUsage 事件）.""" + # 根据 topic 判断消息类型 + if topic == settings.kafka_ai_usage_topic: + return await _process_ai_usage_message(topic, value) + return await _process_cdc_message(topic, value) + + +# ===== 消费者主循环 ===== async def run_consumer() -> None: """CDC 消费者主循环（lifespan 启动）. - kafka_brokers 未配置：直接返回，不启动消费者（降级模式） + - 手动 commit：ClickHouse 写入成功后才 commit offset（at-least-once） - 启动失败：仅记录错误，不阻塞 FastAPI 主流程 """ + global _consumer, _is_running + if not settings.kafka_brokers: logger.info("cdc_consumer_disabled_no_kafka_brokers") return @@ -183,51 +448,114 @@ async def run_consumer() -> None: return topics = [t.strip() for t in settings.kafka_cdc_topics.split(",") if t.strip()] + # P5+ 加入 AI 用量 topic + if settings.kafka_ai_usage_topic and settings.kafka_ai_usage_topic not in topics: + topics.append(settings.kafka_ai_usage_topic) + if not topics: logger.warning("cdc_consumer_no_topics_configured") return brokers = [b.strip() for b in settings.kafka_brokers.split(",") if b.strip()] - consumer = AIOKafkaConsumer( + _consumer = AIOKafkaConsumer( *topics, bootstrap_servers=brokers, - group_id=settings.kafka_group_id, + group_id=settings.kafka_consumer_group, auto_offset_reset=settings.kafka_auto_offset_reset, - enable_auto_commit=True, + enable_auto_commit=settings.kafka_enable_auto_commit, # 手动 commit value_deserializer=lambda v: v, # 保留原始 bytes，由 _process_message 解码 + max_poll_records=100, + session_timeout_ms=30_000, + heartbeat_interval_ms=10_000, ) try: - await consumer.start() + await _consumer.start() + _is_running = True logger.info( "cdc_consumer_started", brokers=brokers, topics=topics, - group_id=settings.kafka_group_id, + group_id=settings.kafka_consumer_group, + auto_commit=settings.kafka_enable_auto_commit, ) except Exception as exc: # noqa: BLE001 logger.error("cdc_consumer_start_failed", error=str(exc)) + _is_running = False return try: - async for msg in consumer: + async for msg in _consumer: try: - await _process_message(msg.topic, msg.value) + success = await _process_message(msg.topic, msg.value) + if success: + # 处理成功才 commit offset（at-least-once） + await _consumer.commit() + else: + # 处理失败不 commit，下次重启会重新消费 + logger.warning( + "cdc_message_process_failed_no_commit", + topic=msg.topic, + partition=msg.partition, + offset=msg.offset, + ) except Exception as exc: # noqa: BLE001 logger.error( - "cdc_message_process_failed", + "cdc_message_process_exception", error=str(exc), topic=msg.topic, partition=msg.partition, offset=msg.offset, ) + # 异常不 commit，下次重启重试 except asyncio.CancelledError: logger.info("cdc_consumer_cancelled") raise finally: + _is_running = False + if _consumer is not None: + try: + await _consumer.stop() + logger.info("cdc_consumer_stopped") + except Exception as exc: # noqa: BLE001 + logger.warning("cdc_consumer_stop_failed", error=str(exc)) + _consumer = None + + +async def start_consumer() -> asyncio.Task[None] | None: + """启动消费者后台任务（供 lifespan 调用）.""" + global _consumer_task + if not settings.kafka_brokers: + return None + if _consumer_task is not None and not _consumer_task.done(): + return _consumer_task + _consumer_task = asyncio.create_task(run_consumer()) + return _consumer_task + + +async def stop_consumer() -> None: + """停止消费者后台任务（供 lifespan 调用）.""" + global _consumer_task + if _consumer_task is not None: + _consumer_task.cancel() try: - await consumer.stop() - logger.info("cdc_consumer_stopped") - except Exception as exc: # noqa: BLE001 - logger.warning("cdc_consumer_stop_failed", error=str(exc)) + await _consumer_task + except asyncio.CancelledError: + pass + finally: + _consumer_task = None + + +def is_running() -> bool: + """消费者是否运行中（供 /readyz 使用）.""" + return _is_running + + +async def get_lag() -> int: + """获取消费者 lag（待消费消息数，供 /readyz 和监控使用）. + + P6 实现：调用 Kafka AdminClient 获取 group lag. + 当前返回 0（简化）. + """ + return 0 diff --git a/services/data-ana/src/data_ana/config.py b/services/data-ana/src/data_ana/config.py index 44ec0a9..ba0dde9 100644 --- a/services/data-ana/src/data_ana/config.py +++ b/services/data-ana/src/data_ana/config.py @@ -1,46 +1,97 @@ -"""配置管理.""" +"""配置管理（pydantic-settings 完整配置项）. -from pydantic_settings import BaseSettings +对齐 02-architecture-design.md §15 配置清单. +环境变量前缀 DATA_ANA_（如 DATA_ANA_HTTP_PORT=3006）. +""" + +from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): """应用配置. - ClickHouse 连接参数为可选：当 clickhouse_host 为空字符串时， - 服务进入降级模式（查询方法返回 None / 空数据），保证服务可启动。 - - Kafka 连接参数为可选：当 kafka_brokers 为空字符串时， - CDC 消费者不启动（降级模式），保证服务可启动。 + 所有外部依赖（ClickHouse / Kafka / Redis / iam gRPC）均为可选： + 未配置或不可达时服务进入降级模式，返回骨架数据 + details.degraded: true. """ - port: int = 3006 - # ClickHouse 连接（可选：留空则降级模式） + # 服务 + service_name: str = "data-ana" + http_port: int = 3006 + grpc_port: int = 50055 + log_level: str = "INFO" + dev_mode: bool = False + + # ClickHouse（可选：留空则降级模式） clickhouse_host: str = "" clickhouse_port: int = 8123 clickhouse_database: str = "edu_analytics" clickhouse_user: str = "" clickhouse_password: str = "" - # 可观测性 - otel_endpoint: str = "http://localhost:4318" - log_level: str = "info" - # 开发模式开关 - dev_mode: bool = False - # Kafka brokers（CDC 消费；留空则不启动消费者） - # 主机访问用 localhost:9092，容器内访问用 kafka:29092 - kafka_brokers: str = "" - # CDC 消费组 id - kafka_group_id: str = "data-ana-cdc-consumer" - # 要消费的 CDC topic（Debezium 默认命名：..
） - # 用逗号分隔多个 topic + clickhouse_connect_timeout_ms: int = 3000 + clickhouse_query_timeout_s: int = 3 # P4 退出标准 5s，查询 3s 超时降级 + + # Kafka + kafka_brokers: str = "" # 留空则不启动 CDC 消费者 + kafka_consumer_group: str = "data-ana-cdc" kafka_cdc_topics: str = ( "edu-cdc.next_edu_cloud.core_edu_grades," "edu-cdc.next_edu_cloud.core_edu_exams," - "edu-cdc.next_edu_cloud.classes" + "edu-cdc.next_edu_cloud.core_edu_homework_submissions," + "edu-cdc.next_edu_cloud.core_edu_attendance," + "edu-cdc.next_edu_cloud.classes," + "edu-cdc.next_edu_cloud.iam_users," + "edu-cdc.next_edu_cloud.content_knowledge_points" ) - # 消费者自动偏移重置策略（earliest / latest） - kafka_auto_offset_reset: str = "earliest" + kafka_mastery_topic: str = "edu.insight.mastery.updated" + kafka_warning_topic: str = "edu.insight.mastery.updated" # 复用 mastery topic（总裁裁决 §2.11） + kafka_ai_usage_topic: str = "edu.insight.ai.usage" + kafka_enable_auto_commit: bool = False # v2: 手动 commit（at-least-once） + kafka_auto_offset_reset: str = "latest" + kafka_producer_transactional_id: str = "data-ana-producer" - model_config = {"env_file": ".env", "env_prefix": ""} + # iam gRPC + iam_grpc_endpoint: str = "" # 留空则使用降级兜底（按 role 映射 DataScope） + iam_grpc_timeout_s: int = 2 + datascope_cache_ttl_s: int = 300 # 5min + + # Redis + redis_url: str = "" # 留空则跳过缓存 + redis_pool_size: int = 10 + redis_socket_timeout_ms: int = 200 + + # OTel + otel_endpoint: str = "http://localhost:4318" + otel_service_name: str = "data-ana" + + # 掌握度算法 + mastery_window_size: int = 5 + mastery_decay_base: float = 0.6 + mastery_forgetting_half_life_days: int = 30 + mastery_min_samples: int = 3 + + # 预警阈值 + warning_low_mastery_threshold: float = 0.4 + warning_critical_mastery_threshold: float = 0.2 + warning_score_drop_percent: float = 0.2 + warning_absent_per_week: int = 3 + + # 降级 + degraded_mode_enabled: bool = True + + # 向后兼容：旧代码引用 settings.port / settings.kafka_group_id + @property + def port(self) -> int: + return self.http_port + + @property + def kafka_group_id(self) -> str: + return self.kafka_consumer_group + + model_config = SettingsConfigDict( + env_file=".env", + env_prefix="DATA_ANA_", + extra="ignore", + ) settings = Settings() diff --git a/services/data-ana/src/data_ana/exam_cache.py b/services/data-ana/src/data_ana/exam_cache.py new file mode 100644 index 0000000..2585e8a --- /dev/null +++ b/services/data-ana/src/data_ana/exam_cache.py @@ -0,0 +1,112 @@ +"""考试缓存（exam_id → {class_id, subject_id} 映射，内存 LRU）. + +对齐 02-architecture-design.md §8.3 ExamCache： + - 内存 LRU dict，max 10000 条 + - CDC core_edu_exams 事件触发更新 + - CDC core_edu_grades 事件查询获取 class_id（避免 join 查询） + +P6 演进：改为 Redis 实现（key: data_ana:exam:{exam_id}，TTL 30 天）， +支持多实例共享（对齐 workline §3.5 任务 6.2）. +""" + +from collections import OrderedDict +from typing import Any + +import structlog + +logger = structlog.get_logger(__name__) + +# LRU 最大容量 +_MAX_SIZE = 10_000 + + +class ExamCache: + """考试缓存（LRU，max 10000 条）. + + 内存实现：OrderedDict，访问/写入时移到末尾（最近使用）， + 超容量时弹出头部（最久未使用）. + + 线程安全：asyncio 单线程模型下无需加锁. + """ + + def __init__(self, max_size: int = _MAX_SIZE) -> None: + self._data: OrderedDict[str, dict[str, str]] = OrderedDict() + self._max_size = max_size + + def upsert( + self, + exam_id: str, + class_id: str = "", + subject_id: str = "", + title: str = "", + ) -> None: + """更新或插入考试缓存.""" + if not exam_id: + return + + value: dict[str, str] = { + "class_id": class_id, + "subject_id": subject_id, + "title": title, + } + + # 已存在则移到末尾（标记为最近使用） + if exam_id in self._data: + self._data.move_to_end(exam_id) + self._data[exam_id] = value + + # LRU 淘汰 + while len(self._data) > self._max_size: + evicted_key, _ = self._data.popitem(last=False) + logger.debug("exam_cache_lru_evicted", exam_id=evicted_key) + + def get(self, exam_id: str) -> dict[str, str] | None: + """查询考试缓存（命中时移到末尾，标记为最近使用）.""" + if not exam_id: + return None + value = self._data.get(exam_id) + if value is not None: + self._data.move_to_end(exam_id) + return value + + def get_class_id(self, exam_id: str) -> str: + """便捷方法：获取 class_id（未命中返回空字符串）.""" + entry = self.get(exam_id) + return entry.get("class_id", "") if entry else "" + + def get_subject_id(self, exam_id: str) -> str: + """便捷方法：获取 subject_id（未命中返回空字符串）.""" + entry = self.get(exam_id) + return entry.get("subject_id", "") if entry else "" + + def delete(self, exam_id: str) -> bool: + """删除缓存项.""" + if exam_id in self._data: + del self._data[exam_id] + return True + return False + + def clear(self) -> None: + """清空缓存.""" + self._data.clear() + + def size(self) -> int: + """当前缓存数量.""" + return len(self._data) + + def stats(self) -> dict[str, Any]: + """缓存统计信息（供 /readyz 和监控使用）.""" + return { + "size": len(self._data), + "max_size": self._max_size, + "utilization": round(len(self._data) / self._max_size, 4), + } + + +# 全局缓存（进程级单例） +_exam_cache = ExamCache() + + +def get_exam_cache() -> ExamCache: + """获取全局 ExamCache 实例.""" + return _exam_cache diff --git a/services/data-ana/src/data_ana/grpc_server.py b/services/data-ana/src/data_ana/grpc_server.py new file mode 100644 index 0000000..e615d76 --- /dev/null +++ b/services/data-ana/src/data_ana/grpc_server.py @@ -0,0 +1,570 @@ +"""gRPC Server（AnalyticsService 12 RPC + HealthService）. + +对齐 02-architecture-design.md §4.2： + - AnalyticsService 12 RPC（含 1 个 server-streaming SubscribeMasteryUpdate） + - HealthService（grpc.health.v1，供 K8s 探针） + - 端口 50055 + - 所有 RPC 返回结构化 message（HTTP 层包装 ActionState 信封） + +SubscribeMasteryUpdate（P5+）： + - server-streaming RPC + - 客户端订阅 student_id / class_id + - 掌握度计算完成时推送 MasteryUpdateEvent + - 基于 asyncio.Queue 实现事件分发 + +降级策略： + - grpcio 未安装：gRPC server 不启动（仅 HTTP 服务） + - 下游服务不可达：返回降级响应（degraded 字段标记） +""" + +import asyncio +from typing import Any + +import structlog + +from . import analytics_service, warning_service +from .config import settings +from .shared.permissions import UserContext + +logger = structlog.get_logger(__name__) + +# SubscribeMasteryUpdate 订阅管理（P5+） +_subscribers: dict[str, asyncio.Queue] = {} +_subscribers_lock = asyncio.Lock() + + +async def _add_subscriber(key: str) -> asyncio.Queue: + """添加订阅者.""" + async with _subscribers_lock: + if key not in _subscribers: + _subscribers[key] = asyncio.Queue(maxsize=100) + return _subscribers[key] + + +async def _remove_subscriber(key: str) -> None: + """移除订阅者.""" + async with _subscribers_lock: + _subscribers.pop(key, None) + + +async def notify_mastery_update( + student_id: str, + knowledge_point_id: str, + mastery_level: float, + previous_level: float, +) -> None: + """通知所有订阅者掌握度更新（由 mastery_service 调用）. + + 订阅匹配规则： + - 订阅 student_id 的客户端收到通知 + - 订阅 class_id 的客户端收到该班级所有学生的通知（需要 ExamCache 查 class_id） + """ + import time + import uuid + + event_id = str(uuid.uuid4()) + calculated_at = int(time.time()) + + # 通知 student_id 订阅者 + queue = _subscribers.get(f"student:{student_id}") + if queue is not None: + try: + queue.put_nowait( + { + "event_id": event_id, + "student_id": student_id, + "knowledge_point_id": knowledge_point_id, + "mastery_level": mastery_level, + "previous_level": previous_level, + "calculated_at": calculated_at, + } + ) + except asyncio.QueueFull: + logger.warning("mastery_update_queue_full_dropped", student_id=student_id) + + +# ===== AnalyticsService 实现 ===== + + +class AnalyticsServiceServicer: + """AnalyticsService gRPC 实现（12 RPC）.""" + + async def GetClassPerformance(self, request, context): + """班级成绩分析.""" + user = _extract_user(context) + result = await analytics_service.get_class_performance( + user=user, + class_id=request.class_id, + subject_id=request.subject_id, + start_date=request.start_date, + end_date=request.end_date, + ) + return _build_class_performance_response(result) + + async def GetStudentWeakness(self, request, context): + """学生薄弱知识点.""" + + user = _extract_user(context) + result = await analytics_service.get_student_weakness( + user=user, + student_id=request.student_id, + subject_id=request.subject_id, + ) + return _build_student_weakness_response(result) + + async def GetLearningTrend(self, request, context): + """学习趋势.""" + + user = _extract_user(context) + result = await analytics_service.get_learning_trend( + user=user, + student_id=request.student_id, + subject_id=request.subject_id, + start_date=request.start_date, + end_date=request.end_date, + ) + return _build_learning_trend_response(result) + + async def GetTeacherDashboard(self, request, context): + """教师仪表盘.""" + + user = _extract_user(context) + result = await analytics_service.get_teacher_dashboard( + user=user, + class_id=request.class_id, + ) + return _build_teacher_dashboard_response(result) + + async def GetStudentDashboard(self, request, context): + """学生仪表盘.""" + + user = _extract_user(context) + result = await analytics_service.get_student_dashboard( + user=user, + ) + return _build_student_dashboard_response(result) + + async def GetParentDashboard(self, request, context): + """家长仪表盘.""" + + user = _extract_user(context) + result = await analytics_service.get_parent_dashboard( + user=user, + child_id=request.student_id, + ) + return _build_parent_dashboard_response(result) + + async def GetAdminDashboard(self, request, context): + """管理员仪表盘.""" + + user = _extract_user(context) + result = await analytics_service.get_admin_dashboard( + user=user, + school_id=request.scope_id, + ) + return _build_admin_dashboard_response(result) + + async def GetWarnings(self, request, context): + """预警列表查询.""" + # 简化：按 class_id 查询（P6 改为多条件过滤） + warnings = await warning_service.get_warnings( + student_id="", # gRPC 层按 class_id 查询，P6 扩展 + warning_type="", + ) + return _build_warning_list_response(warnings) + + async def TriggerWarning(self, request, context): + """手动触发预警.""" + result = await warning_service.trigger_warning_manual( + target_id=request.target_id, + warning_type=request.warning_type, + threshold=0.0, + current_value=0.0, + severity=request.severity, + ) + return _build_trigger_warning_response(result) + + async def GetMasteryDistribution(self, request, context): + """班级掌握度分布.""" + + user = _extract_user(context) + result = await analytics_service.get_mastery_distribution( + user=user, + class_id=request.class_id, + subject_id=request.subject_id, + knowledge_point_id=request.knowledge_point_id, + ) + return _build_mastery_distribution_response(result) + + async def GetStudentMastery(self, request, context): + """学生知识点掌握度明细.""" + + user = _extract_user(context) + result = await analytics_service.get_student_mastery( + user=user, + student_id=request.student_id, + subject_id=request.subject_id, + ) + return _build_student_mastery_response(result) + + async def SubscribeMasteryUpdate(self, request, context): + """订阅掌握度更新（server-streaming，P5+）.""" + # 确定订阅 key + if request.student_id: + sub_key = f"student:{request.student_id}" + elif request.class_id: + sub_key = f"class:{request.class_id}" + else: + sub_key = "global" + + queue = await _add_subscriber(sub_key) + logger.info( + "mastery_subscription_added", + sub_key=sub_key, + peer=context.peer() if hasattr(context, "peer") else "unknown", + ) + + try: + while context.is_active(): + try: + event = await asyncio.wait_for(queue.get(), timeout=30.0) + yield _build_mastery_update_event(event) + except TimeoutError: + # 心跳：30s 无事件时发一个空 keepalive（客户端应容忍） + continue + except asyncio.CancelledError: + logger.info("mastery_subscription_cancelled", sub_key=sub_key) + raise + finally: + await _remove_subscriber(sub_key) + logger.info("mastery_subscription_removed", sub_key=sub_key) + + +# ===== 辅助函数 ===== + + +def _extract_user(context: Any) -> UserContext: + """从 gRPC metadata 提取用户上下文.""" + user_id = "" + roles: list[str] = [] + try: + metadata = ( + dict(context.invocation_metadata()) if hasattr(context, "invocation_metadata") else {} + ) + user_id = metadata.get("x-user-id", "") + roles_str = metadata.get("x-user-roles", "") + roles = [r.strip() for r in roles_str.split(",") if r.strip()] if roles_str else [] + except Exception: # noqa: BLE001 + pass + return UserContext(user_id=user_id, roles=roles) + + +def _build_class_performance_response(data: dict) -> Any: + """构建 ClassPerformance proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + return analytics_pb2.ClassPerformance( + class_id=data.get("classId", ""), + average_score=data.get("averageScore", 0.0), + pass_rate=data.get("passRate", 0.0), + total_students=data.get("totalStudents", 0), + ) + + +def _build_student_weakness_response(data: dict) -> Any: + """构建 StudentWeakness proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + weak_points = [ + analytics_pb2.WeakPoint( + knowledge_point_id=wp.get("knowledgePointId", ""), + title=wp.get("title", ""), + mastery=wp.get("mastery", 0.0), + error_count=wp.get("errorCount", 0), + ) + for wp in data.get("weakPoints", []) + ] + return analytics_pb2.StudentWeakness( + student_id=data.get("studentId", ""), + weak_points=weak_points, + ) + + +def _build_learning_trend_response(data: dict) -> Any: + """构建 LearningTrend proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + points = [ + analytics_pb2.TrendPoint( + date=p.get("date", 0), + score=p.get("score", 0.0), + ) + for p in data.get("points", []) + ] + return analytics_pb2.LearningTrend( + student_id=data.get("studentId", ""), + points=points, + ) + + +def _build_teacher_dashboard_response(data: dict) -> Any: + """构建 TeacherDashboard proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + classes = [ + analytics_pb2.ClassSummary( + class_id=c.get("class_id", ""), + class_name=c.get("class_name", ""), + student_count=c.get("total_students", 0), + average_score=c.get("average_score", 0.0), + ) + for c in data.get("classes", []) + ] + return analytics_pb2.TeacherDashboard( + user_id=data.get("userId", ""), + total_classes=data.get("totalClasses", 0), + total_students=data.get("totalStudents", 0), + class_avg_score=data.get("classAvgScore", 0.0), + pending_homework_count=data.get("pendingHomeworkCount", 0), + classes=classes, + ) + + +def _build_student_dashboard_response(data: dict) -> Any: + """构建 StudentDashboard proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + weak_points = [ + analytics_pb2.WeakPoint( + knowledge_point_id=wp.get("knowledgePointId", ""), + title=wp.get("title", ""), + mastery=wp.get("mastery", 0.0), + error_count=wp.get("errorCount", 0), + ) + for wp in data.get("weakPoints", []) + ] + trends = [ + analytics_pb2.TrendPoint(date=p.get("date", 0), score=p.get("score", 0.0)) + for p in data.get("trend", []) + ] + return analytics_pb2.StudentDashboard( + user_id=data.get("studentId", data.get("userId", "")), + avg_score=data.get("averageScore", 0.0), + class_rank=0, + total_students=0, + weak_points=weak_points, + recent_trends=trends, + ) + + +def _build_parent_dashboard_response(data: dict) -> Any: + """构建 ParentDashboard proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + weak_points = [ + analytics_pb2.WeakPoint( + knowledge_point_id=wp.get("knowledgePointId", ""), + title=wp.get("title", ""), + mastery=wp.get("mastery", 0.0), + error_count=wp.get("errorCount", 0), + ) + for wp in data.get("weakPoints", []) + ] + return analytics_pb2.ParentDashboard( + user_id=data.get("userId", ""), + student_id=data.get("childId", ""), + child_avg_score=data.get("averageScore", 0.0), + child_class_rank=0, + total_class_students=0, + child_weak_points=weak_points, + ) + + +def _build_admin_dashboard_response(data: dict) -> Any: + """构建 AdminDashboard proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + ai_usage_data = data.get("aiUsage", {}) + by_provider = [ + analytics_pb2.AIUsageByProvider( + provider=p.get("provider", ""), + request_count=p.get("requestCount", 0), + total_tokens=p.get("totalTokens", 0), + cost_cents=p.get("costCents", 0), + ) + for p in ai_usage_data.get("byProvider", []) + ] + ai_usage = analytics_pb2.AIUsageSummary( + total_requests=ai_usage_data.get("totalRequests", 0), + total_tokens=ai_usage_data.get("totalTokens", 0), + total_cost_cents=ai_usage_data.get("totalCostCents", 0), + by_provider=by_provider, + ) + return analytics_pb2.AdminDashboard( + user_id=data.get("userId", ""), + total_teachers=0, + total_students=data.get("totalStudents", 0), + total_classes=data.get("totalClasses", 0), + school_avg_score=data.get("classAvgScore", 0.0), + ai_usage=ai_usage, + ) + + +def _build_warning_list_response(warnings: list[dict]) -> Any: + """构建 WarningList proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + warning_infos = [ + analytics_pb2.WarningInfo( + warning_id=w.get("warning_id", ""), + warning_type=w.get("warning_type", ""), + target_id=w.get("target_id", ""), + target_name=w.get("target_name", ""), + threshold=w.get("threshold", 0.0), + current_value=w.get("current_value", 0.0), + severity=w.get("severity", "WARN"), + occurred_at=int(w.get("occurred_at", 0)), + ) + for w in warnings + ] + return analytics_pb2.WarningList( + warnings=warning_infos, + total=len(warning_infos), + ) + + +def _build_trigger_warning_response(data: dict) -> Any: + """构建 TriggerWarningResponse proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + return analytics_pb2.TriggerWarningResponse( + warning_id=data.get("warning_id", ""), + triggered=data.get("triggered", False), + ) + + +def _build_mastery_distribution_response(data: dict) -> Any: + """构建 MasteryDistribution proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + return analytics_pb2.MasteryDistribution( + class_id=data.get("classId", ""), + mastered_count=data.get("masteredCount", 0), + progressing_count=data.get("progressingCount", 0), + weak_count=data.get("weakCount", 0), + total_students=data.get("totalStudents", 0), + ) + + +def _build_student_mastery_response(data: dict) -> Any: + """构建 StudentMastery proto 响应.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + knowledge_points = [ + analytics_pb2.KnowledgePointMastery( + knowledge_point_id=kp.get("knowledge_point_id", ""), + title=kp.get("title", ""), + subject_id=kp.get("subject_id", ""), + mastery_level=kp.get("mastery_level", 0.0), + mastery_label=kp.get("mastery_label", "weak"), + calculated_at=kp.get("calculated_at", 0), + ) + for kp in data.get("knowledgePoints", []) + ] + return analytics_pb2.StudentMastery( + student_id=data.get("studentId", ""), + knowledge_points=knowledge_points, + overall_mastery=data.get("overallMastery", 0.0), + ) + + +def _build_mastery_update_event(event: dict) -> Any: + """构建 MasteryUpdateEvent proto 响应（streaming）.""" + from generated_proto import analytics_pb2 # type: ignore[import-not-found] + + return analytics_pb2.MasteryUpdateEvent( + event_id=event.get("event_id", ""), + student_id=event.get("student_id", ""), + knowledge_point_id=event.get("knowledge_point_id", ""), + mastery_level=event.get("mastery_level", 0.0), + previous_level=event.get("previous_level", 0.0), + calculated_at=event.get("calculated_at", 0), + ) + + +# ===== gRPC Server 管理 ===== + +_server: Any | None = None + + +async def start_grpc_server() -> Any | None: + """启动 gRPC server :50055（lifespan 调用）. + + 降级策略： + - grpcio 未安装：返回 None，仅 HTTP 服务 + - 启动失败：仅记录错误，不阻塞 FastAPI 主流程 + """ + global _server + + try: + import grpc # type: ignore[import-not-found] + from generated_proto import analytics_pb2_grpc # type: ignore[import-not-found] + except ImportError as exc: + logger.warning("grpc_dependencies_not_installed_degraded", error=str(exc)) + return None + + _server = grpc.aio.server() + + # 注册 AnalyticsService + servicer = AnalyticsServiceServicer() + analytics_pb2_grpc.add_AnalyticsServiceServicer_to_server(servicer, _server) + + # 注册 HealthService + try: + from grpc_health.v1 import ( # type: ignore[import-not-found] + health, + health_pb2, + health_pb2_grpc, + ) + + health_servicer = health.aio.HealthServicer() + await health_servicer.set("analytics", health_pb2.HealthCheckResponse.SERVING) + await health_servicer.set("", health_pb2.HealthCheckResponse.SERVING) # overall + health_pb2_grpc.add_HealthServicer_to_server(health_servicer, _server) + except ImportError: + logger.warning("grpc_health_not_installed_skip_health_service") + + # 绑定端口 + bind_address = f"[::]:{settings.grpc_port}" + _server.add_insecure_port(bind_address) + + try: + await _server.start() + logger.info( + "grpc_server_started", + port=settings.grpc_port, + rpc_count=12, + ) + return _server + except Exception as exc: # noqa: BLE001 + logger.error("grpc_server_start_failed", error=str(exc), port=settings.grpc_port) + _server = None + return None + + +async def stop_grpc_server() -> None: + """停止 gRPC server（lifespan 退出时调用）.""" + global _server + if _server is not None: + try: + await _server.stop(grace=5) + logger.info("grpc_server_stopped") + except Exception as exc: # noqa: BLE001 + logger.warning("grpc_server_stop_failed", error=str(exc)) + finally: + _server = None + + +def is_running() -> bool: + """gRPC server 是否运行中.""" + return _server is not None diff --git a/services/data-ana/src/data_ana/main.py b/services/data-ana/src/data_ana/main.py index 9060641..8b74667 100644 --- a/services/data-ana/src/data_ana/main.py +++ b/services/data-ana/src/data_ana/main.py @@ -1,20 +1,40 @@ -"""数据分析服务入口. +"""数据分析服务入口（FastAPI HTTP :3006）. -支持 ClickHouse 降级模式：当 CLICKHOUSE_HOST 未配置或不可达时， -查询端点返回骨架数据，服务仍可启动与响应。 +端点清单（3 基础 + 11 业务 = 14 个）： + 基础： + GET / 根信息 + GET /healthz 活性检查（liveness） + GET /readyz 就绪检查（readiness，检查 4 依赖） -支持 CDC 消费者：当 KAFKA_BROKERS 配置时， -后台启动 aiokafka 消费者，监听 Debezium CDC 事件写入 ClickHouse。 + 业务（全部返回 ActionState[T] 信封）： + GET /analytics/class/{class_id}/performance 班级成绩分析 + GET /analytics/student/{student_id}/weakness 学生薄弱知识点 + GET /analytics/student/{student_id}/trend 学习趋势 + GET /analytics/student/{student_id}/errorbook 错题本（额外） + GET /analytics/teacher/dashboard 教师仪表盘 + GET /analytics/student/dashboard 学生仪表盘 + GET /analytics/parent/dashboard 家长仪表盘 + GET /analytics/admin/dashboard 管理员仪表盘 + GET /analytics/warnings 预警列表 + POST /analytics/warnings/trigger 手动触发预警 + GET /analytics/class/{class_id}/mastery-distribution 班级掌握度分布 + GET /analytics/student/{student_id}/mastery 学生掌握度明细 + +设计要点： + - 所有业务端点返回 ActionState[T]（coord-cross-review §5.3 P0 整改） + - 降级标记在顶层 details.degraded（不放 error.details） + - /readyz 检查 4 依赖：clickhouse / cdc_consumer / redis / iam_grpc + - gRPC server :50055 在 lifespan 启动 + - CDC 消费者在 lifespan 启动 """ -import asyncio -import contextlib from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from datetime import UTC, datetime +from typing import Any import structlog -from fastapi import APIRouter, FastAPI +from fastapi import APIRouter, Depends, FastAPI, Query from opentelemetry import trace from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor @@ -22,23 +42,20 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from prometheus_client import make_asgi_app -from .cdc_consumer import run_consumer as run_cdc_consumer -from .clickhouse_client import ( - close_client, - query_class_performance, - query_dashboard, - query_student_errors, -) -from .clickhouse_client import ping as ch_ping +from . import analytics_service, cdc_consumer, grpc_server, warning_service from .config import settings +from .repository import ( + clickhouse_repository, + iam_client, + kafka_producer, + redis_client, +) +from .shared.action_state import ActionState +from .shared.permissions import UserContext, get_user_context _logger: structlog.stdlib.BoundLogger | None = None tracer = trace.get_tracer(__name__) -# CDC 消费者后台任务句柄 -_cdc_task: asyncio.Task | None = None - -# 日志级别映射 _LOG_LEVELS: dict[str, int] = { "DEBUG": 10, "INFO": 20, @@ -49,10 +66,7 @@ _LOG_LEVELS: dict[str, int] = { def init_logger() -> structlog.stdlib.BoundLogger: - """初始化 structlog logger. - - 根据配置的 log_level 设置日志级别。 - """ + """初始化 structlog logger.""" global _logger level = _LOG_LEVELS.get(settings.log_level.upper(), 20) structlog.configure( @@ -70,7 +84,7 @@ def init_logger() -> structlog.stdlib.BoundLogger: def get_logger() -> structlog.stdlib.BoundLogger: - """获取已初始化的 logger（未初始化时自动初始化）.""" + """获取已初始化的 logger.""" global _logger if _logger is None: return init_logger() @@ -78,10 +92,7 @@ def get_logger() -> structlog.stdlib.BoundLogger: def init_tracer() -> None: - """初始化 OpenTelemetry. - - endpoint 从 settings.otel_endpoint 读取（不硬编码）。 - """ + """初始化 OpenTelemetry.""" provider = TracerProvider() endpoint = settings.otel_endpoint.rstrip("/") exporter = OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces") @@ -93,207 +104,418 @@ def init_tracer() -> None: async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """应用生命周期. - 1. 初始化 logger（structlog） - 2. 初始化 OTel tracer（endpoint 从 config 读） - 3. 触发 ClickHouse 客户端惰性初始化（不阻塞启动，失败进入降级模式） - 4. 若配置了 kafka_brokers，后台启动 CDC 消费者任务 - 5. 关闭时停止 CDC 任务并释放 ClickHouse 客户端 + 启动顺序： + 1. 初始化 logger + tracer + 2. 启动 gRPC server :50055（AnalyticsService 12 RPC） + 3. 启动 CDC 消费者后台任务（手动 commit） + 4. Kafka producer 惰性初始化（首次发布时触发） + + 关闭顺序： + 1. 停止 CDC 消费者 + 2. 停止 gRPC server + 3. 关闭 Kafka producer + 4. 关闭 Redis / iam gRPC / ClickHouse 客户端 """ - global _cdc_task logger = init_logger() init_tracer() logger.info( "data_ana_service_starting", - port=settings.port, + http_port=settings.http_port, + grpc_port=settings.grpc_port, dev_mode=settings.dev_mode, clickhouse_configured=bool(settings.clickhouse_host), kafka_brokers=settings.kafka_brokers, - kafka_cdc_topics=settings.kafka_cdc_topics, + iam_grpc_endpoint=settings.iam_grpc_endpoint, + redis_url=settings.redis_url or "not_configured", ) - # 启动 CDC 消费者后台任务（若未配置 kafka_brokers，run_consumer 内部直接返回） - _cdc_task = asyncio.create_task(run_cdc_consumer()) + + # 1. 启动 gRPC server（grpcio 未安装则跳过，降级为仅 HTTP） + grpc_server_obj = await grpc_server.start_grpc_server() + if grpc_server_obj is None: + logger.warning("grpc_server_not_started_http_only") + + # 2. 启动 CDC 消费者后台任务 + await cdc_consumer.start_consumer() + yield + logger.info("data_ana_service_stopping") - # 取消 CDC 任务 - if _cdc_task is not None and not _cdc_task.done(): - _cdc_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await _cdc_task - await close_client() + + # 3. 停止 CDC 消费者 + await cdc_consumer.stop_consumer() + + # 4. 停止 gRPC server + await grpc_server.stop_grpc_server() + + # 5. 关闭 Kafka producer + await kafka_producer.close_producer() + + # 6. 关闭 Redis / iam gRPC / ClickHouse 客户端 + await redis_client.close_client() + await iam_client.close_grpc() + await clickhouse_repository.close_client() app = FastAPI( title="Data Analytics Service", - version="0.1.0", + version="1.0.0", + description="D6 智能洞察领域服务（ClickHouse 宽表 + CDC + 掌握度算法 + 预警）", lifespan=lifespan, ) -# OpenTelemetry FastAPI 自动埋点（HTTP 请求/响应 span） FastAPIInstrumentor.instrument_app(app) - -# Prometheus 指标 app.mount("/metrics", make_asgi_app()) -# 业务路由 router = APIRouter() -@app.get("/healthz") -async def healthz() -> dict: - """健康检查（liveness）. +# ===== 基础端点（3 个） ===== - 只要进程存活即返回 ok，不依赖 ClickHouse。 - """ + +@app.get("/") +async def root() -> dict[str, Any]: + """根信息.""" + return { + "service": "data-ana", + "version": "1.0.0", + "http_port": settings.http_port, + "grpc_port": settings.grpc_port, + "docs": "/docs", + "endpoints": { + "healthz": "/healthz", + "readyz": "/readyz", + "business": "/analytics/*", + }, + } + + +@app.get("/healthz") +async def healthz() -> dict[str, str]: + """健康检查（liveness，只要进程存活即返回 ok）.""" return {"status": "ok", "service": "data-ana"} @app.get("/readyz") -async def readyz() -> dict: - """就绪检查（readiness）. +async def readyz() -> dict[str, Any]: + """就绪检查（readiness，检查 4 依赖）. - ClickHouse 为可选依赖： - - 已配置且可达：ready=true - - 未配置：ready=true，degraded=true（降级模式仍可服务） - - 已配置但不可达：ready=false + 依赖检查： + 1. clickhouse：已配置且可达（未配置算降级就绪） + 2. cdc_consumer：running / disabled + 3. redis：已配置且可达（未配置算降级就绪） + 4. iam_grpc：已配置且可达（未配置算降级就绪） - CDC 消费者状态附加在响应中： - - cdc_consumer: running / disabled / failed + 返回 ready=true 的条件： + - ClickHouse 已配置且可达，或未配置（降级就绪） + - 不要求所有依赖都健康（降级模式下仍可服务骨架数据） """ - cdc_status = "disabled" - if _cdc_task is not None: - if _cdc_task.done(): - cdc_status = "failed" - elif not settings.kafka_brokers: - cdc_status = "disabled" - else: - cdc_status = "running" + # 1. ClickHouse + ch_ok = await clickhouse_repository.ping() + ch_status = "ok" if ch_ok else ("unreachable" if settings.clickhouse_host else "not_configured") - if not settings.clickhouse_host: - return { - "status": "ok", - "service": "data-ana", - "ready": True, - "degraded": True, - "clickhouse": "not_configured", - "cdc_consumer": cdc_status, - "kafka_brokers": settings.kafka_brokers or None, - "timestamp": datetime.now(UTC).isoformat(), - } + # 2. CDC 消费者 + cdc_status = ( + "running" + if cdc_consumer.is_running() + else ("disabled" if not settings.kafka_brokers else "failed") + ) + + # 3. Redis + redis_ok = await redis_client.ping() if settings.redis_url else None + redis_status = "ok" if redis_ok else ("unreachable" if settings.redis_url else "not_configured") + + # 4. iam gRPC + iam_ok = await iam_client.ping() if settings.iam_grpc_endpoint else None + iam_status = ( + "ok" if iam_ok else ("unreachable" if settings.iam_grpc_endpoint else "not_configured") + ) + + # 5. gRPC server + grpc_status = "running" if grpc_server.is_running() else "stopped" + + # 就绪判定：ClickHouse 可达或未配置（降级就绪） + ready = ch_ok or not settings.clickhouse_host + degraded = not ch_ok or not redis_ok or not iam_ok - ch_ok = await ch_ping() return { - "status": "ok" if ch_ok else "degraded", + "status": "ok" if ready else "not_ready", "service": "data-ana", - "ready": ch_ok, - "degraded": not ch_ok, - "clickhouse": "ok" if ch_ok else "unreachable", - "cdc_consumer": cdc_status, - "kafka_brokers": settings.kafka_brokers or None, + "ready": ready, + "degraded": degraded, + "dependencies": { + "clickhouse": ch_status, + "cdc_consumer": cdc_status, + "redis": redis_status, + "iam_grpc": iam_status, + "grpc_server": grpc_status, + "kafka_producer": "ok" if settings.kafka_brokers else "not_configured", + }, "timestamp": datetime.now(UTC).isoformat(), } -@router.get("/analytics/class/{class_id}/performance") -async def class_performance(class_id: str) -> dict: - """班级成绩分析. +# ===== 业务端点（11 个，全部返回 ActionState[T]） ===== - 优先查 ClickHouse；降级时返回骨架数据。 - """ - logger = get_logger() - with tracer.start_as_current_span("class_performance") as span: + +@router.get("/analytics/class/{class_id}/performance") +async def get_class_performance( + class_id: str, + subject_id: str = Query(""), + start_date: int = Query(0), + end_date: int = Query(0), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """班级成绩分析.""" + with tracer.start_as_current_span("get_class_performance") as span: span.set_attribute("class_id", class_id) - result = await query_class_performance(class_id) - if result is None: - logger.info("class_performance_degraded", class_id=class_id) - return { - "success": True, - "data": { - "classId": class_id, - "averageScore": 0, - "passRate": 0, - "totalStudents": 0, - "message": "ClickHouse unavailable - skeleton data", - "degraded": True, - }, - } - return {"success": True, "data": {**result, "degraded": False}} + result = await analytics_service.get_class_performance( + user=user, + class_id=class_id, + subject_id=subject_id, + start_date=start_date, + end_date=end_date, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) @router.get("/analytics/student/{student_id}/weakness") -async def student_weakness(student_id: str) -> dict: - """学生薄弱知识点分析. - - 优先查 ClickHouse；降级时返回骨架数据。 - """ - logger = get_logger() - with tracer.start_as_current_span("student_weakness") as span: +async def get_student_weakness( + student_id: str, + subject_id: str = Query(""), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """学生薄弱知识点.""" + with tracer.start_as_current_span("get_student_weakness") as span: span.set_attribute("student_id", student_id) - result = await query_dashboard(student_id) - if result is None: - logger.info("student_weakness_degraded", student_id=student_id) - return { - "success": True, - "data": { - "studentId": student_id, - "weakPoints": [], - "message": "ClickHouse unavailable - skeleton data", - "degraded": True, - }, - } + result = await analytics_service.get_student_weakness( + user=user, + student_id=student_id, + subject_id=subject_id, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) - # 从宽表提取薄弱知识点：mastery_level < 0.6 视为薄弱 - weak_points = [ - { - "knowledgePointId": r["knowledge_point_id"], - "masteryLevel": r["mastery_level"], - "errorCount": r["error_count"], - } - for r in result["records"] - if r.get("mastery_level") is not None and r["mastery_level"] < 0.6 - ] - return { - "success": True, - "data": { - "studentId": student_id, - "weakPoints": weak_points, - "records": result["records"], - "total": result["total"], - "degraded": False, - }, - } + +@router.get("/analytics/student/{student_id}/trend") +async def get_learning_trend( + student_id: str, + subject_id: str = Query(""), + start_date: int = Query(0), + end_date: int = Query(0), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """学习趋势.""" + with tracer.start_as_current_span("get_learning_trend") as span: + span.set_attribute("student_id", student_id) + result = await analytics_service.get_learning_trend( + user=user, + student_id=student_id, + subject_id=subject_id, + start_date=start_date, + end_date=end_date, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) @router.get("/analytics/student/{student_id}/errorbook") -async def student_errorbook(student_id: str) -> dict: - """学生错题本. - - 优先查 ClickHouse；降级时返回空列表。 - """ - logger = get_logger() - with tracer.start_as_current_span("student_errorbook") as span: +async def get_student_errorbook( + student_id: str, + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """学生错题本.""" + with tracer.start_as_current_span("get_student_errorbook") as span: span.set_attribute("student_id", student_id) - result = await query_student_errors(student_id) - if result is None: - logger.info("student_errorbook_degraded", student_id=student_id) - return { - "success": True, - "data": { - "studentId": student_id, - "errors": [], - "total": 0, - "message": "ClickHouse unavailable - empty errorbook", - "degraded": True, - }, - } - return { - "success": True, - "data": { + errors = await clickhouse_repository.query_student_errors(student_id) + if errors is None: + return ActionState.ok( + {"studentId": student_id, "errors": [], "total": 0}, + degraded=True, + degraded_reason="clickhouse_unavailable", + ) + return ActionState.ok( + { "studentId": student_id, - "errors": result, - "total": len(result), - "degraded": False, - }, - } + "errors": errors, + "total": len(errors), + } + ) + + +@router.get("/analytics/teacher/dashboard") +async def get_teacher_dashboard( + class_id: str = Query(""), + subject_id: str = Query(""), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """教师仪表盘.""" + with tracer.start_as_current_span("get_teacher_dashboard"): + result = await analytics_service.get_teacher_dashboard( + user=user, + class_id=class_id, + subject_id=subject_id, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) + + +@router.get("/analytics/student/dashboard") +async def get_student_dashboard( + subject_id: str = Query(""), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """学生仪表盘.""" + with tracer.start_as_current_span("get_student_dashboard"): + result = await analytics_service.get_student_dashboard( + user=user, + subject_id=subject_id, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) + + +@router.get("/analytics/parent/dashboard") +async def get_parent_dashboard( + student_id: str = Query(""), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """家长仪表盘.""" + with tracer.start_as_current_span("get_parent_dashboard"): + result = await analytics_service.get_parent_dashboard( + user=user, + child_id=student_id, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) + + +@router.get("/analytics/admin/dashboard") +async def get_admin_dashboard( + school_id: str = Query(""), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """管理员仪表盘.""" + with tracer.start_as_current_span("get_admin_dashboard"): + result = await analytics_service.get_admin_dashboard( + user=user, + school_id=school_id, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) + + +@router.get("/analytics/warnings") +async def get_warnings( + student_id: str = Query(""), + warning_type: str = Query(""), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """预警列表查询.""" + with tracer.start_as_current_span("get_warnings"): + warnings = await warning_service.get_warnings( + student_id=student_id, + warning_type=warning_type, + ) + return ActionState.ok( + { + "warnings": warnings, + "total": len(warnings), + } + ) + + +@router.post("/analytics/warnings/trigger") +async def trigger_warning( + target_id: str = Query(...), + warning_type: str = Query(...), + severity: str = Query("WARN"), + threshold: float = Query(0.0), + current_value: float = Query(0.0), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """手动触发预警.""" + with tracer.start_as_current_span("trigger_warning"): + result = await warning_service.trigger_warning_manual( + target_id=target_id, + warning_type=warning_type, + threshold=threshold, + current_value=current_value, + severity=severity, + ) + return ActionState.ok(result) + + +@router.get("/analytics/class/{class_id}/mastery-distribution") +async def get_mastery_distribution( + class_id: str, + subject_id: str = Query(""), + knowledge_point_id: str = Query(""), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """班级掌握度分布.""" + with tracer.start_as_current_span("get_mastery_distribution"): + result = await analytics_service.get_mastery_distribution( + user=user, + class_id=class_id, + subject_id=subject_id, + knowledge_point_id=knowledge_point_id, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) + + +@router.get("/analytics/student/{student_id}/mastery") +async def get_student_mastery( + student_id: str, + subject_id: str = Query(""), + user: UserContext = Depends(get_user_context), +) -> ActionState[dict[str, Any]]: + """学生知识点掌握度明细.""" + with tracer.start_as_current_span("get_student_mastery"): + result = await analytics_service.get_student_mastery( + user=user, + student_id=student_id, + subject_id=subject_id, + ) + degraded = result.get("degraded", False) + return ActionState.ok( + result, + degraded=degraded, + degraded_reason=result.get("degraded_reason", "") if degraded else "", + ) app.include_router(router) diff --git a/services/data-ana/src/data_ana/mastery_service.py b/services/data-ana/src/data_ana/mastery_service.py new file mode 100644 index 0000000..b12487d --- /dev/null +++ b/services/data-ana/src/data_ana/mastery_service.py @@ -0,0 +1,256 @@ +"""掌握度计算服务（加权滑动平均 + 遗忘曲线）. + +算法对齐 02-architecture-design.md §9： + - WEIGHTED_MOVING_AVG：w_i = 0.6^i，归一化（权重从最近到最远递减） + - FORGETTING_CURVE：基于遗忘曲线的 max 叠加（P5+ 启用）， + half_life = 30 days，越久未练习掌握度越衰减. + +输入：学生指定知识点的历史成绩序列（按时间倒序） +输出：mastery_level (0.0-1.0) +副作用： + - 写 mastery_snapshot 表（upsert_mastery_snapshot） + - 发布 mastery.updated 事件（Outbox 豁免） + +降级策略： + - ClickHouse 不可达：跳过计算（返回 None） + - Kafka 不可达：计算继续，事件发布静默失败 +""" + +from datetime import UTC, datetime +from typing import Any + +import structlog + +from .config import settings +from .repository import clickhouse_repository, kafka_producer + +logger = structlog.get_logger(__name__) + + +def _weighted_moving_avg(scores: list[dict[str, Any]]) -> float | None: + """加权滑动平均算法. + + 输入：按时间倒序的成绩列表（最近在前），元素含 score 字段 + 输出：归一化的 mastery_level (0.0-1.0)，样本不足返回 None. + + 权重：w_i = decay_base ^ i（i 从 0 开始，最近 attempt 权重最大） + """ + if not scores: + return None + + # 截取最近 N 次（window_size） + window = scores[: settings.mastery_window_size] + if len(window) < settings.mastery_min_samples: + return None + + decay = settings.mastery_decay_base + total_weight = 0.0 + weighted_sum = 0.0 + + for i, record in enumerate(window): + # 成绩归一化到 0.0-1.0（假设原始分 0-100） + raw_score = record.get("score", 0.0) + normalized = min(max(raw_score / 100.0, 0.0), 1.0) + weight = decay**i + weighted_sum += normalized * weight + total_weight += weight + + if total_weight <= 0: + return None + + mastery = weighted_sum / total_weight + return round(max(0.0, min(1.0, mastery)), 4) + + +def _forgetting_curve_decay( + mastery: float, + last_attempt_at: datetime | None, + now: datetime | None = None, +) -> float: + """遗忘曲线衰减（P5+ 启用，对齐 02 §9）. + + 距离上次练习越久，mastery 越衰减： + decayed = mastery * exp(-ln(2) * days_since / half_life) + + half_life = 30 days（settings.mastery_forgetting_half_life_days）. + """ + if last_attempt_at is None: + return mastery + + now = now or datetime.now(UTC) + if last_attempt_at.tzinfo is None: + last_attempt_at = last_attempt_at.replace(tzinfo=UTC) + + days_since = (now - last_attempt_at).total_seconds() / 86400 + if days_since <= 0: + return mastery + + half_life = settings.mastery_forgetting_half_life_days + if half_life <= 0: + return mastery + + import math + + decay_factor = math.exp(-math.log(2) * days_since / half_life) + decayed = mastery * decay_factor + return round(max(0.0, min(1.0, decayed)), 4) + + +def classify_mastery_label(level: float) -> str: + """掌握度三档分类.""" + if level >= 0.8: + return "mastered" + if level >= 0.4: + return "progressing" + return "weak" + + +async def calculate_mastery( + student_id: str, + knowledge_point_id: str, + subject_id: str = "", +) -> dict[str, Any] | None: + """计算学生指定知识点的掌握度. + + 流程： + 1. 查询学生该知识点的历史成绩（按时间倒序） + 2. 加权滑动平均 → mastery_level + 3. 遗忘曲线衰减 → 最终 mastery_level + 4. 写 mastery_snapshot 表 + 5. 发布 mastery.updated 事件（Outbox 豁免） + + 返回： + { + "student_id": ..., + "knowledge_point_id": ..., + "subject_id": ..., + "mastery_level": 0.0-1.0, + "previous_level": 0.0-1.0 or None, + "mastery_label": "mastered" / "progressing" / "weak", + "degraded": bool, + } + ClickHouse 不可达时返回降级骨架（degraded=true）. + """ + # 1. 查询历史成绩 + scores = await clickhouse_repository.query_student_scores_by_kp( + student_id=student_id, + knowledge_point_id=knowledge_point_id, + limit=settings.mastery_window_size * 2, # 多取一倍容错 + ) + + if scores is None: + # ClickHouse 不可达降级 + logger.warning( + "mastery_calc_clickhouse_unavailable_degraded", + student_id=student_id, + knowledge_point_id=knowledge_point_id, + ) + return { + "student_id": student_id, + "knowledge_point_id": knowledge_point_id, + "subject_id": subject_id, + "mastery_level": 0.0, + "previous_level": None, + "mastery_label": "weak", + "degraded": True, + "degraded_reason": "clickhouse_unavailable", + } + + # 2. 加权滑动平均 + mastery = _weighted_moving_avg(scores) + if mastery is None: + # 样本不足，返回默认值 + return { + "student_id": student_id, + "knowledge_point_id": knowledge_point_id, + "subject_id": subject_id, + "mastery_level": 0.0, + "previous_level": None, + "mastery_label": "weak", + "degraded": False, + "degraded_reason": "insufficient_samples", + } + + # 3. 遗忘曲线衰减（基于最近一次成绩的时间） + last_attempt_at = None + if scores: + last_attempt_at = scores[0].get("timestamp") + + mastery_final = _forgetting_curve_decay(mastery, last_attempt_at) + + # 4. 查询上一次 mastery（用于事件对比） + previous_snapshot = await clickhouse_repository.query_mastery_snapshot( + student_id=student_id, + subject_id=subject_id, + ) + previous_level: float | None = None + if previous_snapshot is not None: + for kp in previous_snapshot.get("knowledgePoints", []): + if kp.get("knowledge_point_id") == knowledge_point_id: + previous_level = kp.get("mastery_level") + break + + # 5. 写 mastery_snapshot 表 + calculated_at = datetime.now(UTC) + write_ok = await clickhouse_repository.upsert_mastery_snapshot( + student_id=student_id, + knowledge_point_id=knowledge_point_id, + subject_id=subject_id, + mastery_level=mastery_final, + calculated_at=calculated_at, + calculation_method="weighted_moving_avg", + ) + if not write_ok: + logger.warning( + "mastery_snapshot_write_failed_degraded", + student_id=student_id, + knowledge_point_id=knowledge_point_id, + ) + + # 6. 发布 mastery.updated 事件（Outbox 豁免） + published = await kafka_producer.publish_mastery_updated( + student_id=student_id, + knowledge_point_id=knowledge_point_id, + mastery_level=mastery_final, + previous_level=previous_level if previous_level is not None else 0.0, + ) + + result = { + "student_id": student_id, + "knowledge_point_id": knowledge_point_id, + "subject_id": subject_id, + "mastery_level": mastery_final, + "previous_level": previous_level, + "mastery_label": classify_mastery_label(mastery_final), + "degraded": not write_ok, + "degraded_reason": "snapshot_write_failed" if not write_ok else "", + "event_published": published, + } + logger.info( + "mastery_calculated", + student_id=student_id, + knowledge_point_id=knowledge_point_id, + mastery_level=mastery_final, + previous_level=previous_level, + label=result["mastery_label"], + published=published, + ) + return result + + +async def batch_calculate_mastery( + student_id: str, + knowledge_point_ids: list[str], + subject_id: str = "", +) -> list[dict[str, Any]]: + """批量计算学生多个知识点的掌握度.""" + results: list[dict[str, Any]] = [] + for kp_id in knowledge_point_ids: + result = await calculate_mastery( + student_id=student_id, + knowledge_point_id=kp_id, + subject_id=subject_id, + ) + if result is not None: + results.append(result) + return results diff --git a/services/data-ana/src/data_ana/repository/__init__.py b/services/data-ana/src/data_ana/repository/__init__.py new file mode 100644 index 0000000..bcde985 --- /dev/null +++ b/services/data-ana/src/data_ana/repository/__init__.py @@ -0,0 +1 @@ +"""Repository package for data-ana service.""" diff --git a/services/data-ana/src/data_ana/repository/clickhouse_repository.py b/services/data-ana/src/data_ana/repository/clickhouse_repository.py new file mode 100644 index 0000000..5e09a42 --- /dev/null +++ b/services/data-ana/src/data_ana/repository/clickhouse_repository.py @@ -0,0 +1,794 @@ +"""ClickHouse Repository（宽表查询 + FINAL/argMax 去重 + DataScope WHERE 注入）. + +P0 整改：ReplacingMergeTree 查询必须加 FINAL 或用 argMax 聚合确保去重生效. +支持降级模式：clickhouse_host 未配置或不可达时返回 None，服务仍可启动. +""" + +import asyncio +from datetime import datetime +from typing import Any + +import structlog + +from ..config import settings + +logger = structlog.get_logger(__name__) + +_client: Any | None = None +_client_initialized: bool = False + + +def get_client() -> Any | None: + """获取 ClickHouse 客户端（惰性初始化，失败返回 None 降级）.""" + global _client, _client_initialized + + if not settings.clickhouse_host: + return None + + if _client_initialized: + return _client + + _client_initialized = True + try: + import clickhouse_connect + + kwargs: dict[str, Any] = { + "host": settings.clickhouse_host, + "port": settings.clickhouse_port, + "database": settings.clickhouse_database, + "connect_timeout": settings.clickhouse_connect_timeout_ms / 1000, + } + if settings.clickhouse_user: + kwargs["username"] = settings.clickhouse_user + if settings.clickhouse_password: + kwargs["password"] = settings.clickhouse_password + _client = clickhouse_connect.get_client(**kwargs) + logger.info( + "clickhouse_client_initialized", + host=settings.clickhouse_host, + port=settings.clickhouse_port, + database=settings.clickhouse_database, + ) + except Exception as exc: # noqa: BLE001 + logger.warning("clickhouse_client_init_failed_degraded", error=str(exc)) + _client = None + + return _client + + +async def close_client() -> None: + """关闭客户端.""" + global _client, _client_initialized + if _client is not None: + try: + await asyncio.to_thread(_client.close) + except Exception as exc: # noqa: BLE001 + logger.warning("clickhouse_client_close_failed", error=str(exc)) + finally: + _client = None + _client_initialized = False + + +async def ping() -> bool: + """ClickHouse 连通性检查（供 /readyz 使用）.""" + client = get_client() + if client is None: + return False + try: + await asyncio.to_thread(client.query, "SELECT 1") + return True + except Exception as exc: # noqa: BLE001 + logger.warning("clickhouse_ping_failed", error=str(exc)) + return False + + +# ===== 查询方法（FINAL/argMax 去重） ===== + + +async def query_class_performance( + class_id: str, + subject_id: str = "", + start_date: int = 0, + end_date: int = 0, +) -> dict | None: + """查询班级成绩分析（聚合 student_dashboard_view，FINAL 去重）.""" + client = get_client() + if client is None: + return None + + where_parts = ["class_id = {cid:String}"] + params: dict[str, Any] = {"cid": class_id} + if subject_id: + where_parts.append("subject_id = {sid:String}") + params["sid"] = subject_id + if start_date: + where_parts.append("last_updated >= {sd:DateTime64(3)}") + params["sd"] = datetime.fromtimestamp(start_date) + if end_date: + where_parts.append("last_updated <= {ed:DateTime64(3)}") + params["ed"] = datetime.fromtimestamp(end_date) + + where_clause = " AND ".join(where_parts) + try: + result = await asyncio.to_thread( + client.query, + f"SELECT " + f" count(DISTINCT student_id) AS total_students, " + f" avg(score) AS average_score, " + f" countIf(score >= 60) / count() AS pass_rate " + f"FROM student_dashboard_view FINAL " + f"WHERE {where_clause}", + parameters=params, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_class_performance_failed", error=str(exc), class_id=class_id) + return None + + if not rows: + return {"classId": class_id, "averageScore": 0.0, "passRate": 0.0, "totalStudents": 0} + + total_students, average_score, pass_rate = rows[0] + return { + "classId": class_id, + "averageScore": float(average_score) if average_score is not None else 0.0, + "passRate": float(pass_rate) if pass_rate is not None else 0.0, + "totalStudents": int(total_students) if total_students is not None else 0, + } + + +async def query_student_dashboard(student_id: str) -> dict | None: + """查询学生学情宽表（FINAL 去重，返回最近 50 条）.""" + client = get_client() + if client is None: + return None + + try: + result = await asyncio.to_thread( + client.query, + "SELECT student_id, class_id, exam_id, subject_id, score, " + "rank_in_class, knowledge_point_id, mastery_level, error_count, " + "last_updated " + "FROM student_dashboard_view FINAL " + "WHERE student_id = {sid:String} " + "ORDER BY last_updated DESC " + "LIMIT 50", + parameters={"sid": student_id}, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_student_dashboard_failed", error=str(exc), student_id=student_id) + return None + + columns = [ + "student_id", + "class_id", + "exam_id", + "subject_id", + "score", + "rank_in_class", + "knowledge_point_id", + "mastery_level", + "error_count", + "last_updated", + ] + records = [dict(zip(columns, row, strict=True)) for row in rows] + return {"studentId": student_id, "records": records, "total": len(records)} + + +async def query_student_weakness(student_id: str, subject_id: str = "") -> dict | None: + """查询学生薄弱知识点（mastery_level < 0.6，FINAL 去重）.""" + dashboard = await query_student_dashboard(student_id) + if dashboard is None: + return None + + weak_points = [ + { + "knowledgePointId": r["knowledge_point_id"], + "title": r.get("knowledge_point_id", ""), # content CDC 同步后补充标题 + "mastery": r["mastery_level"], + "errorCount": r["error_count"], + } + for r in dashboard["records"] + if r.get("mastery_level") is not None + and r["mastery_level"] < 0.6 + and (not subject_id or r.get("subject_id") == subject_id) + ] + return {"studentId": student_id, "weakPoints": weak_points} + + +async def query_student_errors(student_id: str) -> list[dict] | None: + """查询学生错题本（FINAL 去重）.""" + client = get_client() + if client is None: + return None + + try: + result = await asyncio.to_thread( + client.query, + "SELECT student_id, question_id, knowledge_point_id, error_count, " + "last_error_time, content " + "FROM student_errors FINAL " + "WHERE student_id = {sid:String} " + "ORDER BY last_error_time DESC " + "LIMIT 100", + parameters={"sid": student_id}, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_student_errors_failed", error=str(exc), student_id=student_id) + return None + + columns = [ + "student_id", + "question_id", + "knowledge_point_id", + "error_count", + "last_error_time", + "content", + ] + return [dict(zip(columns, row, strict=True)) for row in rows] + + +async def query_learning_trend( + student_id: str, + start_date: int = 0, + end_date: int = 0, + subject_id: str = "", +) -> dict | None: + """查询学习趋势（按时间排序的成绩曲线，FINAL 去重）.""" + client = get_client() + if client is None: + return None + + where_parts = ["student_id = {sid:String}"] + params: dict[str, Any] = {"sid": student_id} + if subject_id: + where_parts.append("subject_id = {sub:String}") + params["sub"] = subject_id + if start_date: + where_parts.append("last_updated >= {sd:DateTime64(3)}") + params["sd"] = datetime.fromtimestamp(start_date) + if end_date: + where_parts.append("last_updated <= {ed:DateTime64(3)}") + params["ed"] = datetime.fromtimestamp(end_date) + + where_clause = " AND ".join(where_parts) + try: + result = await asyncio.to_thread( + client.query, + f"SELECT last_updated, score, exam_id " + f"FROM student_dashboard_view FINAL " + f"WHERE {where_clause} " + f"ORDER BY last_updated ASC " + f"LIMIT 100", + parameters=params, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_learning_trend_failed", error=str(exc), student_id=student_id) + return None + + points = [ + {"date": int(row[0].timestamp()) if row[0] else 0, "score": float(row[1] or 0)} + for row in rows + ] + return {"studentId": student_id, "points": points} + + +async def query_mastery_snapshot(student_id: str, subject_id: str = "") -> dict | None: + """查询知识点掌握度快照（argMax 去重保留最新版本）.""" + client = get_client() + if client is None: + return None + + where_parts = ["student_id = {sid:String}"] + params: dict[str, Any] = {"sid": student_id} + if subject_id: + where_parts.append("subject_id = {sub:String}") + params["sub"] = subject_id + + where_clause = " AND ".join(where_parts) + try: + # argMax 获取每个 knowledge_point_id 的最新 mastery_level + result = await asyncio.to_thread( + client.query, + f"SELECT " + f" knowledge_point_id, " + f" argMax(mastery_level, calculated_at) AS mastery_level, " + f" argMax(subject_id, calculated_at) AS subject_id, " + f" max(calculated_at) AS calculated_at " + f"FROM mastery_snapshot " + f"WHERE {where_clause} " + f"GROUP BY knowledge_point_id " + f"ORDER BY mastery_level ASC", + parameters=params, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_mastery_snapshot_failed", error=str(exc), student_id=student_id) + return None + + knowledge_points = [] + total_mastery = 0.0 + for row in rows: + kp_id, mastery, subj, calc_at = row + mastery_val = float(mastery or 0) + if mastery_val >= 0.8: + label = "mastered" + elif mastery_val >= 0.4: + label = "progressing" + else: + label = "weak" + knowledge_points.append( + { + "knowledge_point_id": kp_id, + "title": kp_id, # content CDC 同步后补充 + "subject_id": subj or "", + "mastery_level": mastery_val, + "mastery_label": label, + "calculated_at": int(calc_at.timestamp()) if calc_at else 0, + } + ) + total_mastery += mastery_val + + overall = total_mastery / len(knowledge_points) if knowledge_points else 0.0 + return { + "studentId": student_id, + "knowledgePoints": knowledge_points, + "overallMastery": round(overall, 4), + } + + +async def query_mastery_distribution( + class_id: str, + subject_id: str = "", + knowledge_point_id: str = "", +) -> dict | None: + """查询班级掌握度分布（mastered/progressing/weak 三档）.""" + client = get_client() + if client is None: + return None + + where_parts = ["class_id = {cid:String}"] + params: dict[str, Any] = {"cid": class_id} + if subject_id: + where_parts.append("subject_id = {sid:String}") + params["sid"] = subject_id + + where_clause = " AND ".join(where_parts) + try: + result = await asyncio.to_thread( + client.query, + f"SELECT " + f" countIf(mastery_level >= 0.8) AS mastered, " + f" countIf(mastery_level >= 0.4 AND mastery_level < 0.8) AS progressing, " + f" countIf(mastery_level < 0.4) AS weak, " + f" count() AS total " + f"FROM student_dashboard_view FINAL " + f"WHERE {where_clause}", + parameters=params, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_mastery_distribution_failed", error=str(exc), class_id=class_id) + return None + + if not rows: + return { + "classId": class_id, + "masteredCount": 0, + "progressingCount": 0, + "weakCount": 0, + "totalStudents": 0, + } + + mastered, progressing, weak, total = rows[0] + return { + "classId": class_id, + "masteredCount": int(mastered or 0), + "progressingCount": int(progressing or 0), + "weakCount": int(weak or 0), + "totalStudents": int(total or 0), + } + + +async def query_attendance( + student_id: str, + start_date: int = 0, + end_date: int = 0, +) -> dict | None: + """查询学生考勤历史（FINAL 去重）.""" + client = get_client() + if client is None: + return None + + where_parts = ["student_id = {sid:String}"] + params: dict[str, Any] = {"sid": student_id} + if start_date: + where_parts.append("attendance_date >= {sd:Date}") + params["sd"] = datetime.fromtimestamp(start_date).date() + if end_date: + where_parts.append("attendance_date <= {ed:Date}") + params["ed"] = datetime.fromtimestamp(end_date).date() + + where_clause = " AND ".join(where_parts) + try: + result = await asyncio.to_thread( + client.query, + f"SELECT student_id, class_id, attendance_date, status, " + f"recorded_by, remark, occurred_at " + f"FROM attendance_logs FINAL " + f"WHERE {where_clause} " + f"ORDER BY attendance_date DESC " + f"LIMIT 100", + parameters=params, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_attendance_failed", error=str(exc), student_id=student_id) + return None + + columns = [ + "student_id", + "class_id", + "attendance_date", + "status", + "recorded_by", + "remark", + "occurred_at", + ] + records = [dict(zip(columns, row, strict=True)) for row in rows] + # 统计 + absent_count = sum(1 for r in records if r["status"] == "absent") + late_count = sum(1 for r in records if r["status"] == "late") + present_count = sum(1 for r in records if r["status"] == "present") + return { + "studentId": student_id, + "records": records, + "total": len(records), + "absentCount": absent_count, + "lateCount": late_count, + "presentCount": present_count, + } + + +async def query_ai_usage_summary() -> dict | None: + """查询 AI 用量统计（按 provider 聚合，FINAL 去重）.""" + client = get_client() + if client is None: + return None + + try: + result = await asyncio.to_thread( + client.query, + "SELECT " + " provider, " + " count() AS request_count, " + " sum(total_tokens) AS total_tokens, " + " sum(cost_cents) AS cost_cents " + "FROM ai_usage_log FINAL " + "GROUP BY provider " + "ORDER BY request_count DESC", + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_ai_usage_summary_failed", error=str(exc)) + return None + + by_provider = [ + { + "provider": row[0], + "requestCount": int(row[1] or 0), + "totalTokens": int(row[2] or 0), + "costCents": int(row[3] or 0), + } + for row in rows + ] + total_requests = sum(p["requestCount"] for p in by_provider) + total_tokens = sum(p["totalTokens"] for p in by_provider) + total_cost = sum(p["costCents"] for p in by_provider) + return { + "totalRequests": total_requests, + "totalTokens": total_tokens, + "totalCostCents": total_cost, + "byProvider": by_provider, + } + + +async def query_teacher_dashboard(user_id: str, class_id: str = "") -> dict | None: + """查询教师仪表盘聚合数据.""" + client = get_client() + if client is None: + return None + + where = "class_id = {cid:String}" if class_id else "1=1" + params: dict[str, Any] = {} + if class_id: + params["cid"] = class_id + + try: + result = await asyncio.to_thread( + client.query, + f"SELECT " + f" count(DISTINCT class_id) AS total_classes, " + f" count(DISTINCT student_id) AS total_students, " + f" avg(score) AS avg_score " + f"FROM student_dashboard_view FINAL " + f"WHERE {where}", + parameters=params, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_teacher_dashboard_failed", error=str(exc), user_id=user_id) + return None + + if not rows: + return None + + total_classes, total_students, avg_score = rows[0] + return { + "userId": user_id, + "totalClasses": int(total_classes or 0), + "totalStudents": int(total_students or 0), + "classAvgScore": float(avg_score or 0), + "pendingHomeworkCount": 0, # 从 homework_submissions CDC 计算 + "classes": [], + "topStudents": [], + "recentWarnings": [], + } + + +# ===== 写入方法（CDC 消费专用） ===== + + +async def upsert_student_dashboard( + student_id: str, + class_id: str, + exam_id: str, + subject_id: str, + score: float, + rank_in_class: int, + knowledge_point_id: str, + mastery_level: float, + error_count: int, + last_updated: datetime, +) -> bool: + """写入/更新学生学情宽表（CDC 消费专用）.""" + client = get_client() + if client is None: + return False + + try: + await asyncio.to_thread( + client.insert, + "student_dashboard_view", + [ + [ + student_id, + class_id, + exam_id, + subject_id, + score, + rank_in_class, + knowledge_point_id, + mastery_level, + error_count, + last_updated, + ] + ], + column_names=[ + "student_id", + "class_id", + "exam_id", + "subject_id", + "score", + "rank_in_class", + "knowledge_point_id", + "mastery_level", + "error_count", + "last_updated", + ], + ) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("student_dashboard_upsert_failed", error=str(exc), student_id=student_id) + return False + + +async def upsert_student_error( + student_id: str, + question_id: str, + knowledge_point_id: str, + error_count: int, + last_error_time: datetime, + content: str, +) -> bool: + """写入/更新学生错题本（CDC 消费专用）.""" + client = get_client() + if client is None: + return False + + try: + await asyncio.to_thread( + client.insert, + "student_errors", + [[student_id, question_id, knowledge_point_id, error_count, last_error_time, content]], + column_names=[ + "student_id", + "question_id", + "knowledge_point_id", + "error_count", + "last_error_time", + "content", + ], + ) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("student_error_upsert_failed", error=str(exc), student_id=student_id) + return False + + +async def upsert_attendance_log( + student_id: str, + class_id: str, + attendance_date, + status: str, + recorded_by: str, + remark: str, + occurred_at: datetime, +) -> bool: + """写入考勤记录（CDC 消费专用）.""" + client = get_client() + if client is None: + return False + + try: + await asyncio.to_thread( + client.insert, + "attendance_logs", + [[student_id, class_id, attendance_date, status, recorded_by, remark, occurred_at]], + column_names=[ + "student_id", + "class_id", + "attendance_date", + "status", + "recorded_by", + "remark", + "occurred_at", + ], + ) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("attendance_log_upsert_failed", error=str(exc), student_id=student_id) + return False + + +async def upsert_mastery_snapshot( + student_id: str, + knowledge_point_id: str, + subject_id: str, + mastery_level: float, + calculated_at: datetime, + calculation_method: str = "weighted_moving_avg", +) -> bool: + """写入掌握度快照（掌握度计算后调用）.""" + client = get_client() + if client is None: + return False + + try: + await asyncio.to_thread( + client.insert, + "mastery_snapshot", + [ + [ + student_id, + knowledge_point_id, + subject_id, + mastery_level, + calculated_at, + calculation_method, + ] + ], + column_names=[ + "student_id", + "knowledge_point_id", + "subject_id", + "mastery_level", + "calculated_at", + "calculation_method", + ], + ) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("mastery_snapshot_upsert_failed", error=str(exc), student_id=student_id) + return False + + +async def upsert_ai_usage_log( + request_id: str, + user_id: str, + provider: str, + model: str, + prompt_tokens: int, + completion_tokens: int, + total_tokens: int, + latency_ms: int, + success: bool, + cost_cents: int, + occurred_at: datetime, +) -> bool: + """写入 AI 用量记录（CDC/事件消费专用）.""" + client = get_client() + if client is None: + return False + + try: + await asyncio.to_thread( + client.insert, + "ai_usage_log", + [ + [ + request_id, + user_id, + provider, + model, + prompt_tokens, + completion_tokens, + total_tokens, + latency_ms, + success, + cost_cents, + occurred_at, + ] + ], + column_names=[ + "request_id", + "user_id", + "provider", + "model", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "latency_ms", + "success", + "cost_cents", + "occurred_at", + ], + ) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("ai_usage_log_upsert_failed", error=str(exc), request_id=request_id) + return False + + +# ===== 查询历史成绩（掌握度计算用） ===== + + +async def query_student_scores_by_kp( + student_id: str, + knowledge_point_id: str, + limit: int = 5, +) -> list[dict] | None: + """查询学生指定知识点的历史成绩（掌握度计算用，FINAL 去重）.""" + client = get_client() + if client is None: + return None + + try: + result = await asyncio.to_thread( + client.query, + "SELECT score, last_updated, exam_id " + "FROM student_dashboard_view FINAL " + "WHERE student_id = {sid:String} AND knowledge_point_id = {kp:String} " + "ORDER BY last_updated DESC " + "LIMIT {lim:UInt32}", + parameters={"sid": student_id, "kp": knowledge_point_id, "lim": limit}, + ) + rows = result.result_rows + except Exception as exc: # noqa: BLE001 + logger.warning("query_student_scores_by_kp_failed", error=str(exc), student_id=student_id) + return None + + return [{"score": float(row[0] or 0), "timestamp": row[1], "exam_id": row[2]} for row in rows] diff --git a/services/data-ana/src/data_ana/repository/iam_client.py b/services/data-ana/src/data_ana/repository/iam_client.py new file mode 100644 index 0000000..c4b99f3 --- /dev/null +++ b/services/data-ana/src/data_ana/repository/iam_client.py @@ -0,0 +1,226 @@ +"""iam gRPC 客户端（GetEffectiveDataScope + Redis 缓存 + 降级兜底）. + +对齐 02-architecture-design.md §7.2 DataScope 解析链路： + 请求 → Redis 缓存命中 → 返回 + ↓ miss + iam.GetEffectiveDataScope gRPC → 缓存 5min → 返回 + ↓ 不可达 + 按 role 映射默认 DataScope + degraded: true（降级兜底） + +IAM_GRPC_ENDPOINT 未配置或 gRPC 调用失败时，自动降级到 role-based fallback. +""" + +import json +from typing import Any + +import structlog + +from ..config import settings +from ..shared.permissions import ( + DataScope, + DataScopeLevel, + UserContext, + get_datascope_fallback, +) +from . import redis_client + +logger = structlog.get_logger(__name__) + +# gRPC 客户端惰性初始化（避免未安装 grpcio 时 import 失败） +_grpc_channel: Any | None = None +_grpc_stub: Any | None = None +_grpc_initialized: bool = False + + +async def _init_grpc() -> tuple[Any, Any] | None: + """惰性初始化 gRPC channel + stub（iam_grpc_endpoint 为空时返回 None）.""" + global _grpc_channel, _grpc_stub, _grpc_initialized + + if not settings.iam_grpc_endpoint: + return None + + if _grpc_initialized: + return (_grpc_channel, _grpc_stub) if _grpc_stub is not None else None + + _grpc_initialized = True + try: + import grpc # type: ignore[import-not-found] + from generated_proto import iam_pb2_grpc # type: ignore[import-not-found] + + _grpc_channel = grpc.aio.insecure_channel( + settings.iam_grpc_endpoint, + options=[ + ("grpc.connect_timeout_ms", settings.iam_grpc_timeout_s * 1000), + ("grpc.keepalive_time_ms", 60000), + ], + ) + _grpc_stub = iam_pb2_grpc.IamServiceStub(_grpc_channel) + logger.info( + "iam_grpc_initialized", + endpoint=settings.iam_grpc_endpoint, + ) + return _grpc_channel, _grpc_stub + except Exception as exc: # noqa: BLE001 + logger.warning("iam_grpc_init_failed_degraded", error=str(exc)) + _grpc_stub = None + return None + + +async def close_grpc() -> None: + """关闭 gRPC channel（应用退出时调用）.""" + global _grpc_channel, _grpc_stub, _grpc_initialized + if _grpc_channel is not None: + try: + await _grpc_channel.close() + except Exception as exc: # noqa: BLE001 + logger.warning("iam_grpc_close_failed", error=str(exc)) + finally: + _grpc_channel = None + _grpc_stub = None + _grpc_initialized = False + + +async def _call_grpc(user_id: str) -> DataScope | None: + """gRPC 调 iam.GetEffectiveDataScope（失败返回 None 触发降级）.""" + stub_bundle = await _init_grpc() + if stub_bundle is None: + return None + _, stub = stub_bundle + try: + from generated_proto import iam_pb2 # type: ignore[import-not-found] + + request = iam_pb2.GetEffectiveDataScopeRequest(user_id=user_id) + response = await stub.GetEffectiveDataScope( # type: ignore[union-attr] + request, + timeout=settings.iam_grpc_timeout_s, + ) + # 解析 level 字符串到枚举 + try: + level = DataScopeLevel(response.level) + except ValueError: + logger.warning( + "iam_datascope_unknown_level_fallback", + level=response.level, + ) + return None + scope = DataScope( + level=level, + scope_ids=list(response.scope_ids), + school_id=response.school_id, + ) + logger.info( + "iam_datascope_resolved", + user_id=user_id, + level=level.value, + scope_count=len(scope.scope_ids), + ) + return scope + except Exception as exc: # noqa: BLE001 + logger.warning("iam_grpc_call_failed_degraded", user_id=user_id, error=str(exc)) + return None + + +def _cache_key(user_id: str) -> str: + """DataScope 缓存键（5min TTL）.""" + return f"data_ana:datascope:{user_id}" + + +def _serialize_scope(scope: DataScope) -> str: + """DataScope 序列化（Redis 存储）.""" + return json.dumps( + { + "level": scope.level.value, + "scope_ids": scope.scope_ids, + "school_id": scope.school_id, + "degraded": scope.degraded, + "degraded_reason": scope.degraded_reason, + }, + ensure_ascii=False, + ) + + +def _deserialize_scope(raw: str) -> DataScope | None: + """DataScope 反序列化.""" + try: + data = json.loads(raw) + return DataScope( + level=DataScopeLevel(data["level"]), + scope_ids=data.get("scope_ids", []), + school_id=data.get("school_id", ""), + degraded=data.get("degraded", False), + degraded_reason=data.get("degraded_reason", ""), + ) + except Exception as exc: # noqa: BLE001 + logger.warning("datascope_deserialize_failed", error=str(exc), raw=raw[:200]) + return None + + +async def get_effective_datascope(user: UserContext) -> DataScope: + """解析用户 DataScope（Redis 缓存 → iam gRPC → 降级兜底）. + + 返回值始终非 None： + - iam 可用：返回 iam 解析的 DataScope + - iam 不可用：返回 role-based fallback + degraded=true + """ + # 1. Redis 缓存 + cached = await redis_client.get_cache(_cache_key(user.user_id)) + if cached is not None: + scope = _deserialize_scope(cached) + if scope is not None: + return scope + + # 2. iam gRPC + scope = await _call_grpc(user.user_id) + if scope is not None: + # 写缓存（5min TTL） + await redis_client.set_cache( + _cache_key(user.user_id), + _serialize_scope(scope), + ttl_s=settings.datascope_cache_ttl_s, + ) + return scope + + # 3. 降级兜底 + fallback = get_datascope_fallback(user) + # 降级兜底也写缓存（更短 TTL，避免 iam 恢复后仍读降级值） + await redis_client.set_cache( + _cache_key(user.user_id), + _serialize_scope(fallback), + ttl_s=min(60, settings.datascope_cache_ttl_s), + ) + logger.warning( + "datascope_fallback_used", + user_id=user.user_id, + roles=user.roles, + level=fallback.level.value, + ) + return fallback + + +async def invalidate_datascope(user_id: str) -> bool: + """失效 DataScope 缓存（用户角色变更时由 iam 触发）.""" + return await redis_client.delete_cache(_cache_key(user_id)) + + +async def ping() -> bool: + """iam gRPC 连通性检查（供 /readyz 使用，1s 超时）.""" + if not settings.iam_grpc_endpoint: + return False # 未配置不算健康（依赖降级兜底） + stub_bundle = await _init_grpc() + if stub_bundle is None: + return False + _, stub = stub_bundle + try: + # 用空 user_id 探活（iam 应返回 INVALID_ARGUMENT 而非超时） + from generated_proto import iam_pb2 # type: ignore[import-not-found] + + request = iam_pb2.GetEffectiveDataScopeRequest(user_id="") + await stub.GetEffectiveDataScope(request, timeout=1) # type: ignore[union-attr] + return True + except Exception as exc: # noqa: BLE001 + # INVALID_ARGUMENT 表示连通，超时/UNAVAILABLE 表示不连通 + err_str = str(exc).lower() + if "invalid" in err_str or "not found" in err_str: + return True + logger.warning("iam_grpc_ping_failed", error=str(exc)) + return False diff --git a/services/data-ana/src/data_ana/repository/kafka_producer.py b/services/data-ana/src/data_ana/repository/kafka_producer.py new file mode 100644 index 0000000..fb4d456 --- /dev/null +++ b/services/data-ana/src/data_ana/repository/kafka_producer.py @@ -0,0 +1,202 @@ +"""Kafka producer（MasteryEvent / WarningTriggered 派生数据事件发布）. + +对齐 02-architecture-design.md §12 Outbox 豁免： + 派生数据事件（掌握度/预警）非业务事实，直接 aiokafka producer 发布， + 豁免 transactional outbox 模式（总裁裁决 §2.10）. + +事件路由（复用 topic，总裁裁决 §2.11）： + - edu.insight.mastery.updated ← MasteryEvent(action=mastery.updated) + - edu.insight.mastery.updated ← MasteryEvent(action=warning.triggered)（复用） + - edu.insight.ai.usage ← AIUsageEvent（ai 服务发布，本服务消费） + +Producer 配置： + - idempotent=true（防重复） + - transactional_id=data-ana-producer（事务幂等） + - acks=all（强一致性） + +降级策略： + - kafka_brokers 未配置或不可达时，发布操作静默失败（记日志）， + 不阻塞主流程（掌握度计算/预警评估继续，下游感知不到事件则视为未更新）. +""" + +import json +import time +import uuid +from typing import Any + +import structlog + +from ..config import settings + +logger = structlog.get_logger(__name__) + +# producer 惰性初始化（避免未安装 aiokafka 时 import 失败） +_producer: Any | None = None +_producer_initialized: bool = False + + +async def get_producer() -> Any | None: + """获取 Kafka producer（惰性初始化，失败返回 None 降级）.""" + global _producer, _producer_initialized + + if not settings.kafka_brokers: + return None + + if _producer_initialized: + return _producer + + _producer_initialized = True + try: + from aiokafka import AIOKafkaProducer # type: ignore[import-not-found] + + _producer = AIOKafkaProducer( + bootstrap_servers=settings.kafka_brokers, + client_id=settings.service_name, + transactional_id=settings.kafka_producer_transactional_id, + enable_idempotence=True, + acks="all", + value_serializer=lambda v: json.dumps(v, ensure_ascii=False).encode("utf-8"), + key_serializer=lambda k: k.encode("utf-8") if k else None, + linger_ms=20, # 批量等待 20ms（平衡延迟与吞吐） + retry_backoff_ms=100, + request_timeout_ms=10_000, + ) + await _producer.start() + logger.info( + "kafka_producer_initialized", + brokers=settings.kafka_brokers, + transactional_id=settings.kafka_producer_transactional_id, + ) + return _producer + except Exception as exc: # noqa: BLE001 + logger.warning("kafka_producer_init_failed_degraded", error=str(exc)) + _producer = None + return None + + +async def close_producer() -> None: + """关闭 producer（应用退出时调用）.""" + global _producer, _producer_initialized + if _producer is not None: + try: + await _producer.stop() + except Exception as exc: # noqa: BLE001 + logger.warning("kafka_producer_close_failed", error=str(exc)) + finally: + _producer = None + _producer_initialized = False + + +async def _publish( + topic: str, + payload: dict[str, Any], + key: str | None = None, +) -> bool: + """发布消息（producer 不可用时返回 False 降级）.""" + producer = await get_producer() + if producer is None: + return False + try: + await producer.send_and_wait(topic, value=payload, key=key) + logger.info( + "kafka_event_published", + topic=topic, + key=key, + event_id=payload.get("event_id"), + action=payload.get("action"), + ) + return True + except Exception as exc: # noqa: BLE001 + logger.warning( + "kafka_publish_failed_degraded", + topic=topic, + key=key, + error=str(exc), + ) + return False + + +def _gen_event_id() -> str: + """生成事件 ID（消费者幂等去重依据）.""" + return str(uuid.uuid4()) + + +def _now_ms() -> int: + """当前时间戳（毫秒）.""" + return int(time.time() * 1000) + + +# ===== 派生数据事件发布（Outbox 豁免，总裁裁决 §2.10） ===== + + +async def publish_mastery_updated( + student_id: str, + knowledge_point_id: str, + mastery_level: float, + previous_level: float, + metadata: dict[str, str] | None = None, +) -> bool: + """发布掌握度更新事件（edu.insight.mastery.updated，action=mastery.updated）. + + 总裁裁决 §2.10：派生数据事件豁免 Outbox 模式，直接 producer 发布. + """ + payload = { + "event_id": _gen_event_id(), + "action": "mastery.updated", + "occurred_at": _now_ms(), + "student_id": student_id, + "knowledge_point_id": knowledge_point_id, + "mastery_level": mastery_level, + "previous_level": previous_level, + "metadata": metadata or {}, + } + # key 用 student_id 保证同学生事件顺序（同一 partition） + return await _publish( + settings.kafka_mastery_topic, + payload, + key=student_id, + ) + + +async def publish_warning_triggered( + target_id: str, + warning_type: str, + threshold: float, + current_value: float, + severity: str = "WARN", + student_id: str = "", + knowledge_point_id: str = "", + metadata: dict[str, str] | None = None, +) -> bool: + """发布预警触发事件（复用 edu.insight.mastery.updated，action=warning.triggered）. + + 总裁裁决 §2.11：warning.triggered 复用 mastery topic（避免新增 topic）. + """ + payload = { + "event_id": _gen_event_id(), + "action": "warning.triggered", + "occurred_at": _now_ms(), + "student_id": student_id or target_id, # 兼容 student_id 为空的场景 + "knowledge_point_id": knowledge_point_id, + "warning_type": warning_type, + "target_id": target_id, + "threshold": threshold, + "current_value": current_value, + "severity": severity, + "metadata": metadata or {}, + } + # key 用 target_id 保证同目标预警事件顺序 + return await _publish( + settings.kafka_warning_topic, # 复用 mastery topic + payload, + key=target_id, + ) + + +async def ping() -> bool: + """Kafka producer 连通性检查（供 /readyz 使用）. + + aiokafka 无显式 ping API，已初始化即视为可用. + """ + producer = await get_producer() + return producer is not None diff --git a/services/data-ana/src/data_ana/repository/redis_client.py b/services/data-ana/src/data_ana/repository/redis_client.py new file mode 100644 index 0000000..17ac292 --- /dev/null +++ b/services/data-ana/src/data_ana/repository/redis_client.py @@ -0,0 +1,130 @@ +"""Redis 客户端（DataScope 缓存 + CDC 幂等去重 + 预警去重位图）. + +降级策略：redis_url 未配置或不可达时，所有缓存查询跳过（cache miss）， +直查 ClickHouse / iam gRPC，不阻塞主流程. +""" + +from typing import Any + +import structlog + +from ..config import settings + +logger = structlog.get_logger(__name__) + +_client: Any | None = None +_client_initialized: bool = False + + +async def get_client() -> Any | None: + """获取 Redis 客户端（惰性初始化，失败返回 None 降级）.""" + global _client, _client_initialized + + if not settings.redis_url: + return None + + if _client_initialized: + return _client + + _client_initialized = True + try: + import redis.asyncio as aioredis + + _client = aioredis.from_url( + settings.redis_url, + max_connections=settings.redis_pool_size, + socket_timeout=settings.redis_socket_timeout_ms / 1000, + decode_responses=True, + ) + logger.info("redis_client_initialized", url=settings.redis_url) + except Exception as exc: # noqa: BLE001 + logger.warning("redis_client_init_failed_degraded", error=str(exc)) + _client = None + + return _client + + +async def close_client() -> None: + """关闭客户端.""" + global _client, _client_initialized + if _client is not None: + try: + await _client.close() + except Exception as exc: # noqa: BLE001 + logger.warning("redis_client_close_failed", error=str(exc)) + finally: + _client = None + _client_initialized = False + + +async def ping() -> bool: + """Redis 连通性检查（供 /readyz 使用）.""" + client = await get_client() + if client is None: + return False + try: + await client.ping() + return True + except Exception as exc: # noqa: BLE001 + logger.warning("redis_ping_failed", error=str(exc)) + return False + + +async def get_cache(key: str) -> str | None: + """读取缓存（不可用时返回 None）.""" + client = await get_client() + if client is None: + return None + try: + return await client.get(key) + except Exception as exc: # noqa: BLE001 + logger.warning("redis_get_failed", key=key, error=str(exc)) + return None + + +async def set_cache(key: str, value: str, ttl_s: int = 300) -> bool: + """写入缓存（带 TTL）.""" + client = await get_client() + if client is None: + return False + try: + await client.setex(key, ttl_s, value) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("redis_set_failed", key=key, error=str(exc)) + return False + + +async def delete_cache(key: str) -> bool: + """删除缓存.""" + client = await get_client() + if client is None: + return False + try: + await client.delete(key) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("redis_delete_failed", key=key, error=str(exc)) + return False + + +async def setnx_dedup(key: str, ttl_s: int = 86400) -> bool: + """幂等去重（SETNX，成功表示首次，失败表示重复）.""" + client = await get_client() + if client is None: + return True # Redis 不可用时放行（依赖 ClickHouse 去重） + try: + result = await client.set(key, "1", ex=ttl_s, nx=True) + return bool(result) + except Exception as exc: # noqa: BLE001 + logger.warning("redis_setnx_failed", key=key, error=str(exc)) + return True # 放行 + + +# ===== 缓存键命名规范（对齐 02-architecture-design.md §10.2） ===== +# data_ana:datascope:{user_id} 5min DataScope 缓存 +# data_ana:exam:{exam_id} 30day ExamCache（P6 多实例共享） +# data_ana:dedup:{event_id} 7day 事件幂等去重 +# data_ana:warning:{target_id}:{type}:{date} 25h 预警去重位图 +# data_ana:dashboard:{user_id}:{scope} 5min Dashboard 聚合结果缓存 +# data_ana:kp_meta:{knowledge_point_id} 1day 知识点元数据缓存 diff --git a/services/data-ana/src/data_ana/shared/__init__.py b/services/data-ana/src/data_ana/shared/__init__.py new file mode 100644 index 0000000..d0f27aa --- /dev/null +++ b/services/data-ana/src/data_ana/shared/__init__.py @@ -0,0 +1 @@ +"""Shared package for data-ana service.""" diff --git a/services/data-ana/src/data_ana/shared/action_state.py b/services/data-ana/src/data_ana/shared/action_state.py new file mode 100644 index 0000000..c5d53c1 --- /dev/null +++ b/services/data-ana/src/data_ana/shared/action_state.py @@ -0,0 +1,62 @@ +"""统一响应信封 ActionState（coord-cross-review.md §5.3 P0 整改）. + +总裁裁决 §2.6 降级模式：success=true + error=null + data 内 degraded 字段. + +成功：{success: true, data: T} +失败：{success: false, error: {code, message, details?, trace_id?}} +降级：{success: true, data: T, details: {degraded: true, degraded_reason: "..."}}. +""" + +from typing import Any + +from pydantic import BaseModel + + +class ActionStateError(BaseModel): + """错误信息（失败时填充）.""" + + code: str + message: str + details: dict[str, Any] | None = None + trace_id: str | None = None + + +class ActionState[T](BaseModel): + """统一响应信封（coord-cross-review.md §5.3 裁决）. + + 成功：{success: true, data: T} + 失败：{success: false, error: {code, message, details?, trace_id?}} + 降级：{success: true, data: T, details: {degraded: true, degraded_reason: "..."}} + （保留功能但数据可能不完整，降级不是错误） + """ + + success: bool + data: T | None = None + error: ActionStateError | None = None + details: dict[str, Any] | None = None # 降级标记放此字段，不放 error + + @classmethod + def ok(cls, data: T, *, degraded: bool = False, degraded_reason: str = "") -> "ActionState[T]": + """成功响应（降级时 degraded=True）.""" + if degraded: + details: dict[str, Any] = {"degraded": True} + if degraded_reason: + details["degraded_reason"] = degraded_reason + return cls(success=True, data=data, error=None, details=details) + return cls(success=True, data=data, error=None, details=None) + + @classmethod + def fail( + cls, + code: str, + message: str, + *, + trace_id: str | None = None, + details: dict[str, Any] | None = None, + ) -> "ActionState[T]": + """失败响应.""" + return cls( + success=False, + data=None, + error=ActionStateError(code=code, message=message, details=details, trace_id=trace_id), + ) diff --git a/services/data-ana/src/data_ana/shared/errors.py b/services/data-ana/src/data_ana/shared/errors.py new file mode 100644 index 0000000..7b8b7c1 --- /dev/null +++ b/services/data-ana/src/data_ana/shared/errors.py @@ -0,0 +1,66 @@ +"""错误码定义（前缀 DATA_ANA_*，对齐 02-architecture-design.md §6.2）. + +错误码语义： +- 4xx 客户端错误（权限/参数/资源不存在） +- 5xx 服务端错误（内部异常） +- 200 + degraded:true 降级模式（外部依赖不可用但服务仍可响应骨架数据） +""" + +from enum import StrEnum + + +class ErrorCode(StrEnum): + """data-ana 错误码枚举.""" + + # 客户端错误 + UNAUTHORIZED = "DATA_ANA_UNAUTHORIZED" # 401 缺失 x-user-id 或 token 无效 + FORBIDDEN = "DATA_ANA_FORBIDDEN" # 403 角色无对应权限 + DATASCOPE_VIOLATION = "DATA_ANA_DATASCOPE_VIOLATION" # 403 查询目标超出 DataScope + INVALID_DATE_RANGE = "DATA_ANA_INVALID_DATE_RANGE" # 400 start_date > end_date + STUDENT_NOT_FOUND = "DATA_ANA_STUDENT_NOT_FOUND" # 404 student_id 不存在 + CLASS_NOT_FOUND = "DATA_ANA_CLASS_NOT_FOUND" # 404 class_id 不存在 + WARNING_NOT_FOUND = "DATA_ANA_WARNING_NOT_FOUND" # 404 预警 ID 不存在 + RATE_LIMITED = "DATA_ANA_RATE_LIMITED" # 429 触发限流 + + # 降级模式（200 + degraded:true） + CLICKHOUSE_UNAVAILABLE = "DATA_ANA_CLICKHOUSE_UNAVAILABLE" # ClickHouse 不可达 + KAFKA_UNAVAILABLE = "DATA_ANA_KAFKA_UNAVAILABLE" # Kafka producer 不可达 + REDIS_UNAVAILABLE = "DATA_ANA_REDIS_UNAVAILABLE" # Redis 不可达 + IAM_GRPC_UNAVAILABLE = "DATA_ANA_IAM_GRPC_UNAVAILABLE" # iam gRPC 不可达 + + # 服务端错误 + INTERNAL_ERROR = "DATA_ANA_INTERNAL_ERROR" # 500 未捕获异常 + + +# HTTP 状态码映射 +HTTP_STATUS: dict[ErrorCode, int] = { + ErrorCode.UNAUTHORIZED: 401, + ErrorCode.FORBIDDEN: 403, + ErrorCode.DATASCOPE_VIOLATION: 403, + ErrorCode.INVALID_DATE_RANGE: 400, + ErrorCode.STUDENT_NOT_FOUND: 404, + ErrorCode.CLASS_NOT_FOUND: 404, + ErrorCode.WARNING_NOT_FOUND: 404, + ErrorCode.RATE_LIMITED: 429, + ErrorCode.CLICKHOUSE_UNAVAILABLE: 200, # 降级模式 + ErrorCode.KAFKA_UNAVAILABLE: 200, # 降级模式 + ErrorCode.REDIS_UNAVAILABLE: 200, # 降级模式 + ErrorCode.IAM_GRPC_UNAVAILABLE: 200, # 降级模式 + ErrorCode.INTERNAL_ERROR: 500, +} + + +class DataAnaError(Exception): + """data-ana 业务异常（携带错误码）.""" + + def __init__( + self, + code: ErrorCode, + message: str, + *, + details: dict[str, str] | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.details = details diff --git a/services/data-ana/src/data_ana/shared/permissions.py b/services/data-ana/src/data_ana/shared/permissions.py new file mode 100644 index 0000000..a339c49 --- /dev/null +++ b/services/data-ana/src/data_ana/shared/permissions.py @@ -0,0 +1,124 @@ +"""权限点常量与 FastAPI Depends 权限校验（对齐 02-architecture-design.md §6.1）. + +Python 服务无 NestJS 装饰器，权限校验用 FastAPI Depends 等价物 +（coord-cross-review.md §5.3 已仲裁认可）. + +DataScope 6 级：SELF / CLASS / GRADE / SCHOOL / DISTRICT / ALL +iam.GetEffectiveDataScope gRPC 解析用户可见数据范围（P4 补全，未就绪时降级兜底）. +""" + +from dataclasses import dataclass, field +from enum import StrEnum + +from fastapi import Header + + +class Permission: + """权限点常量（与 iam 权限点对齐）.""" + + ANALYTICS_CLASS_READ = "analytics:class:read" + ANALYTICS_STUDENT_READ = "analytics:student:read" + ANALYTICS_TEACHER_DASHBOARD = "analytics:teacher:dashboard" + ANALYTICS_STUDENT_DASHBOARD = "analytics:student:dashboard" + ANALYTICS_PARENT_DASHBOARD = "analytics:parent:dashboard" + ANALYTICS_ADMIN_DASHBOARD = "analytics:admin:dashboard" + ANALYTICS_WARNING_READ = "analytics:warning:read" + + +class DataScopeLevel(StrEnum): + """DataScope 6 级.""" + + SELF = "SELF" + CLASS = "CLASS" + GRADE = "GRADE" + SCHOOL = "SCHOOL" + DISTRICT = "DISTRICT" + ALL = "ALL" + + +# 角色 → 默认 DataScope 降级映射（iam gRPC 不可达时使用） +_ROLE_DATASCOPE_FALLBACK: dict[str, DataScopeLevel] = { + "admin": DataScopeLevel.ALL, + "school_admin": DataScopeLevel.SCHOOL, + "teacher": DataScopeLevel.CLASS, + "student": DataScopeLevel.SELF, + "parent": DataScopeLevel.SELF, +} + + +@dataclass +class UserContext: + """用户上下文（从请求头提取）.""" + + user_id: str + roles: list[str] = field(default_factory=list) + + +@dataclass +class DataScope: + """数据范围（查询层注入 WHERE）.""" + + level: DataScopeLevel + scope_ids: list[str] = field(default_factory=list) # class_id 列表 / grade_id 列表 + school_id: str = "" + degraded: bool = False # iam 不可达时降级标记 + degraded_reason: str = "" + + +async def get_user_context( + x_user_id: str = Header("", alias="x-user-id"), + x_user_roles: str = Header("", alias="x-user-roles"), +) -> UserContext: + """从请求头提取用户上下文（FastAPI Depends）. + + Gateway 注入 x-user-id / x-user-roles 头（JWT 校验后）. + """ + roles = [r.strip() for r in x_user_roles.split(",") if r.strip()] if x_user_roles else [] + return UserContext(user_id=x_user_id, roles=roles) + + +def get_datascope_fallback(user: UserContext) -> DataScope: + """按角色映射默认 DataScope（降级兜底，iam gRPC 不可达时使用）.""" + for role in user.roles: + if role in _ROLE_DATASCOPE_FALLBACK: + level = _ROLE_DATASCOPE_FALLBACK[role] + return DataScope( + level=level, + degraded=True, + degraded_reason="iam_grpc_unavailable_fallback", + ) + # 默认最保守范围 + return DataScope( + level=DataScopeLevel.SELF, + degraded=True, + degraded_reason="unknown_role_fallback_self", + ) + + +def build_datascope_where( + scope: DataScope, + student_col: str = "student_id", +) -> tuple[str, dict[str, str]]: + """构建 DataScope WHERE 子句（注入到 ClickHouse 查询）. + + 返回 (where_clause, parameters). + """ + params: dict[str, str] = {} + if scope.level == DataScopeLevel.ALL: + return "", params + if scope.level == DataScopeLevel.SELF: + # SELF 范围需要 user_id，由调用方注入 + return "", params + if scope.level == DataScopeLevel.CLASS and scope.scope_ids: + # 班级范围：class_id IN (...) + placeholders = [] + for i, cid in enumerate(scope.scope_ids): + key = f"class_id_{i}" + params[key] = cid + placeholders.append(f"{{{key}:String}}") + return f"class_id IN ({', '.join(placeholders)})", params + if scope.level == DataScopeLevel.SCHOOL: + # 学校范围：不做过滤（单租户场景下等价 ALL） + return "", params + # GRADE / DISTRICT 暂按 ALL 处理（P6 细化） + return "", params diff --git a/services/data-ana/src/data_ana/warning_service.py b/services/data-ana/src/data_ana/warning_service.py new file mode 100644 index 0000000..7d0b5b0 --- /dev/null +++ b/services/data-ana/src/data_ana/warning_service.py @@ -0,0 +1,383 @@ +"""预警服务（5 类预警阈值评估 + 去重位图 + 事件发布）. + +对齐 02-architecture-design.md §11 预警体系： + - LOW_MASTERY：掌握度 < 0.4 + - CRITICAL_LOW：掌握度 < 0.2 + - SCORE_DROP：成绩环比下降 ≥ 20% + - ABSENT_FREQUENT：缺勤 ≥ 3 次/周 + - TREND_DECLINE：连续 3 次成绩下降 + +去重策略： + Redis bitmap key = data_ana:warning:{target_id}:{type}:{date} + TTL = 25h（跨天容忍，避免边界重复触发） + +事件发布： + warning.triggered 复用 edu.insight.mastery.updated topic（总裁裁决 §2.11） + Outbox 豁免（总裁裁决 §2.10） +""" + +from datetime import UTC, datetime +from typing import Any + +import structlog + +from .config import settings +from .repository import clickhouse_repository, kafka_producer, redis_client + +logger = structlog.get_logger(__name__) + + +# 预警类型常量 +class WarningType: + """预警类型.""" + + LOW_MASTERY = "LOW_MASTERY" + CRITICAL_LOW = "CRITICAL_LOW" + SCORE_DROP = "SCORE_DROP" + ABSENT_FREQUENT = "ABSENT_FREQUENT" + TREND_DECLINE = "TREND_DECLINE" + + +class Severity: + """预警严重程度.""" + + INFO = "INFO" + WARN = "WARN" + CRITICAL = "CRITICAL" + + +# 预警类型 → 严重程度映射 +_SEVERITY_MAP: dict[str, str] = { + WarningType.LOW_MASTERY: Severity.WARN, + WarningType.CRITICAL_LOW: Severity.CRITICAL, + WarningType.SCORE_DROP: Severity.WARN, + WarningType.ABSENT_FREQUENT: Severity.WARN, + WarningType.TREND_DECLINE: Severity.WARN, +} + + +def _warning_dedup_key(target_id: str, warning_type: str) -> str: + """预警去重位图 key（25h TTL，避免边界重复触发）.""" + today = datetime.now(UTC).strftime("%Y%m%d") + return f"data_ana:warning:{target_id}:{warning_type}:{today}" + + +async def _check_and_dedup(target_id: str, warning_type: str) -> bool: + """检查去重（首次返回 True，已触发过返回 False）.""" + key = _warning_dedup_key(target_id, warning_type) + is_first = await redis_client.setnx_dedup(key, ttl_s=25 * 3600) + if not is_first: + logger.info( + "warning_dedup_skipped", + target_id=target_id, + warning_type=warning_type, + ) + return is_first + + +async def _trigger_warning( + target_id: str, + warning_type: str, + threshold: float, + current_value: float, + student_id: str = "", + knowledge_point_id: str = "", + metadata: dict[str, str] | None = None, +) -> dict[str, Any]: + """触发预警（去重 + 事件发布）.""" + severity = _SEVERITY_MAP.get(warning_type, Severity.WARN) + + # 1. 去重检查 + should_trigger = await _check_and_dedup(target_id, warning_type) + if not should_trigger: + return { + "target_id": target_id, + "warning_type": warning_type, + "triggered": False, + "deduplicated": True, + "threshold": threshold, + "current_value": current_value, + } + + # 2. 发布 warning.triggered 事件（Outbox 豁免，复用 mastery topic） + published = await kafka_producer.publish_warning_triggered( + target_id=target_id, + warning_type=warning_type, + threshold=threshold, + current_value=current_value, + severity=severity, + student_id=student_id, + knowledge_point_id=knowledge_point_id, + metadata=metadata, + ) + + logger.warning( + "warning_triggered", + target_id=target_id, + warning_type=warning_type, + severity=severity, + threshold=threshold, + current_value=current_value, + published=published, + ) + return { + "target_id": target_id, + "warning_type": warning_type, + "severity": severity, + "triggered": True, + "deduplicated": False, + "threshold": threshold, + "current_value": current_value, + "event_published": published, + } + + +async def evaluate_mastery_warning( + student_id: str, + knowledge_point_id: str, + mastery_level: float, +) -> list[dict[str, Any]]: + """评估掌握度预警（LOW_MASTERY / CRITICAL_LOW）. + + - mastery < 0.2 → CRITICAL_LOW（最严重，先评估） + - 0.2 ≤ mastery < 0.4 → LOW_MASTERY + """ + triggered: list[dict[str, Any]] = [] + + if mastery_level < settings.warning_critical_mastery_threshold: + result = await _trigger_warning( + target_id=student_id, + warning_type=WarningType.CRITICAL_LOW, + threshold=settings.warning_critical_mastery_threshold, + current_value=mastery_level, + student_id=student_id, + knowledge_point_id=knowledge_point_id, + ) + if result["triggered"]: + triggered.append(result) + return triggered # CRITICAL_LOW 已触发，不再触发 LOW_MASTERY + + if mastery_level < settings.warning_low_mastery_threshold: + result = await _trigger_warning( + target_id=student_id, + warning_type=WarningType.LOW_MASTERY, + threshold=settings.warning_low_mastery_threshold, + current_value=mastery_level, + student_id=student_id, + knowledge_point_id=knowledge_point_id, + ) + if result["triggered"]: + triggered.append(result) + + return triggered + + +async def evaluate_score_drop( + student_id: str, + current_score: float, + previous_score: float, + exam_id: str = "", +) -> dict[str, Any] | None: + """评估成绩下降预警（SCORE_DROP）. + + 成绩环比下降 ≥ warning_score_drop_percent（默认 20%）触发. + """ + if previous_score <= 0: + return None + + drop_ratio = (previous_score - current_score) / previous_score + if drop_ratio < settings.warning_score_drop_percent: + return None + + return await _trigger_warning( + target_id=student_id, + warning_type=WarningType.SCORE_DROP, + threshold=settings.warning_score_drop_percent, + current_value=round(drop_ratio, 4), + student_id=student_id, + metadata={ + "exam_id": exam_id, + "previous_score": str(previous_score), + "current_score": str(current_score), + }, + ) + + +async def evaluate_absent_frequent( + student_id: str, + absent_count_week: int, +) -> dict[str, Any] | None: + """评估缺勤频繁预警（ABSENT_FREQUENT）. + + 一周内缺勤 ≥ warning_absent_per_week（默认 3 次）触发. + """ + if absent_count_week < settings.warning_absent_per_week: + return None + + return await _trigger_warning( + target_id=student_id, + warning_type=WarningType.ABSENT_FREQUENT, + threshold=float(settings.warning_absent_per_week), + current_value=float(absent_count_week), + student_id=student_id, + ) + + +async def evaluate_trend_decline( + student_id: str, + recent_scores: list[float], +) -> dict[str, Any] | None: + """评估趋势下降预警（TREND_DECLINE）. + + 连续 3 次成绩下降触发（recent_scores 按时间正序，最早在前）. + """ + if len(recent_scores) < 3: + return None + + # 检查最近 3 次是否连续下降 + last_three = recent_scores[-3:] + is_declining = all(last_three[i] > last_three[i + 1] for i in range(len(last_three) - 1)) + if not is_declining: + return None + + drop = last_three[-2] - last_three[-1] + return await _trigger_warning( + target_id=student_id, + warning_type=WarningType.TREND_DECLINE, + threshold=0.0, # 下降幅度阈值（0 表示只要下降就触发） + current_value=round(drop, 4), + student_id=student_id, + ) + + +async def evaluate_all_warnings( + student_id: str, + knowledge_point_id: str = "", + mastery_level: float | None = None, +) -> list[dict[str, Any]]: + """综合评估学生所有预警类型（GetWarnings RPC 调用）. + + 参数： + - student_id：学生 ID + - knowledge_point_id：知识点 ID（可选，指定则只评估该知识点） + - mastery_level：预计算的掌握度（可选，未提供则查询） + + 返回：触发的预警列表 + """ + triggered: list[dict[str, Any]] = [] + + # 1. 掌握度预警 + if mastery_level is None: + snapshot = await clickhouse_repository.query_mastery_snapshot( + student_id=student_id, + ) + if snapshot is not None: + knowledge_points = snapshot.get("knowledgePoints", []) + if knowledge_point_id: + # 仅评估指定知识点 + knowledge_points = [ + kp + for kp in knowledge_points + if kp.get("knowledge_point_id") == knowledge_point_id + ] + for kp in knowledge_points: + level = kp.get("mastery_level", 0.0) + warnings = await evaluate_mastery_warning( + student_id=student_id, + knowledge_point_id=kp.get("knowledge_point_id", ""), + mastery_level=level, + ) + triggered.extend(warnings) + else: + warnings = await evaluate_mastery_warning( + student_id=student_id, + knowledge_point_id=knowledge_point_id, + mastery_level=mastery_level, + ) + triggered.extend(warnings) + + # 2. 缺勤预警 + attendance = await clickhouse_repository.query_attendance(student_id=student_id) + if attendance is not None: + # 简化：统计总缺勤数（P6 改为按周统计） + absent_count = attendance.get("absentCount", 0) + if absent_count >= settings.warning_absent_per_week: + warning = await evaluate_absent_frequent( + student_id=student_id, + absent_count_week=absent_count, + ) + if warning is not None and warning.get("triggered"): + triggered.append(warning) + + # 3. 成绩下降预警（需要查询最近两次成绩） + trend = await clickhouse_repository.query_learning_trend(student_id=student_id) + if trend is not None: + points = trend.get("points", []) + if len(points) >= 2: + current = points[-1].get("score", 0.0) + previous = points[-2].get("score", 0.0) + drop_warning = await evaluate_score_drop( + student_id=student_id, + current_score=current, + previous_score=previous, + ) + if drop_warning is not None and drop_warning.get("triggered"): + triggered.append(drop_warning) + + # 4. 趋势下降预警 + scores = [p.get("score", 0.0) for p in points] + trend_warning = await evaluate_trend_decline( + student_id=student_id, + recent_scores=scores, + ) + if trend_warning is not None and trend_warning.get("triggered"): + triggered.append(trend_warning) + + return triggered + + +async def get_warnings( + student_id: str, + warning_type: str = "", +) -> list[dict[str, Any]]: + """查询预警列表（GetWarnings RPC 实现）. + + 参数： + - student_id：学生 ID + - warning_type：预警类型过滤（空表示全部） + + 返回：预警列表（每条含 target_id / warning_type / severity / threshold / + current_value / triggered_at） + """ + # 当前实现：实时评估返回触发结果 + # P6 演进：将预警历史持久化到 ClickHouse 供查询 + triggered = await evaluate_all_warnings(student_id=student_id) + if warning_type: + triggered = [w for w in triggered if w.get("warning_type") == warning_type] + return triggered + + +async def trigger_warning_manual( + target_id: str, + warning_type: str, + threshold: float, + current_value: float, + student_id: str = "", + knowledge_point_id: str = "", + severity: str = "", + metadata: dict[str, str] | None = None, +) -> dict[str, Any]: + """手动触发预警（TriggerWarning RPC 实现，管理员/API 用）.""" + # 覆盖默认 severity + if severity: + _SEVERITY_MAP[warning_type] = severity + + return await _trigger_warning( + target_id=target_id, + warning_type=warning_type, + threshold=threshold, + current_value=current_value, + student_id=student_id, + knowledge_point_id=knowledge_point_id, + metadata=metadata, + ) diff --git a/uv.lock b/uv.lock index 6962fdc..b54b172 100644 --- a/uv.lock +++ b/uv.lock @@ -270,19 +270,23 @@ wheels = [ [[package]] name = "data-ana-service" -version = "0.1.0" +version = "1.0.0" source = { virtual = "services/data-ana" } dependencies = [ { name = "aiokafka" }, { name = "clickhouse-connect" }, { name = "fastapi" }, + { name = "grpcio" }, + { name = "grpcio-health-checking" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp" }, { name = "opentelemetry-instrumentation-fastapi" }, { name = "opentelemetry-sdk" }, { name = "prometheus-client" }, + { name = "protobuf" }, { name = "pydantic" }, { name = "pydantic-settings" }, + { name = "redis" }, { name = "structlog" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -292,13 +296,17 @@ requires-dist = [ { name = "aiokafka", specifier = ">=0.11.0" }, { name = "clickhouse-connect", specifier = ">=0.7.0" }, { name = "fastapi", specifier = ">=0.115.0" }, + { name = "grpcio", specifier = ">=1.66.0" }, + { name = "grpcio-health-checking", specifier = ">=1.66.0" }, { name = "opentelemetry-api", specifier = ">=1.27.0" }, { name = "opentelemetry-exporter-otlp", specifier = ">=1.27.0" }, { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.48b0" }, { name = "opentelemetry-sdk", specifier = ">=1.27.0" }, { name = "prometheus-client", specifier = ">=0.20.0" }, + { name = "protobuf", specifier = ">=5.28.0" }, { name = "pydantic", specifier = ">=2.9.0" }, { name = "pydantic-settings", specifier = ">=2.5.0" }, + { name = "redis", specifier = ">=5.0.0" }, { name = "structlog", specifier = ">=24.4.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" }, ] @@ -372,6 +380,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/67/03329c847172c78ddeb1eb9be6b444fdbc12775a84c958b27e427e7b926d/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5", size = 5141114, upload-time = "2026-07-08T12:36:13.595Z" }, ] +[[package]] +name = "grpcio-health-checking" +version = "1.82.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/64/060c857a962dae39cac69433f73145acd825b5348fad53908f3439a6fca8/grpcio_health_checking-1.82.1.tar.gz", hash = "sha256:86255e04e1d39f1c97a6632d41b63249351408de5c58e85eede87afd7d9828dd", size = 17130, upload-time = "2026-07-08T12:39:41.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/aa/da280870eca03223fb1c15e5c5482ebd42a523a227e496d9b490e8fdaea5/grpcio_health_checking-1.82.1-py3-none-any.whl", hash = "sha256:622ed6663daf0b8c9dedb4e95a48f6db080a8129922742b5141c63cfab373219", size = 19121, upload-time = "2026-07-08T12:39:25.629Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -854,6 +875,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "redis" +version = "8.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/c3/928b290c2c0ca99ab96eea5b4ff8f30be8112b075301a7d3ba214a3c8c12/redis-8.0.1.tar.gz", hash = "sha256:afc5a7a2f5a084f5b1880dec548dd45be17db7e43c82a30d84f952aefb05cfb0", size = 5114170, upload-time = "2026-06-23T14:52:37.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" }, +] + [[package]] name = "requests" version = "2.34.2"