feat(data-ana): implement complete CDC pipeline MySQL to ClickHouse

Debezium + Kafka + aiokafka consumer with table routing.

E2E verified: MySQL INSERT to ClickHouse upsert.
This commit is contained in:
SpecialX
2026-07-09 13:02:59 +08:00
parent 958b17c9d8
commit 1f901c5b20
10 changed files with 595 additions and 43 deletions

View File

@@ -5,6 +5,7 @@
保证服务在 ClickHouse 不可用时仍可启动并响应骨架数据。
"""
from datetime import datetime
from typing import Any
import structlog
@@ -216,3 +217,127 @@ async def ping() -> bool:
except Exception as exc: # noqa: BLE001
logger.warning("clickhouse_ping_failed", error=str(exc))
return False
async def upsert_student_dashboard(
student_id: str,
class_id: str,
exam_id: str,
subject_id: str,
score: float,
rank_in_class: int,
knowledge_point_id: str,
mastery_level: float,
error_count: int,
last_updated: datetime,
) -> bool:
"""写入/更新学生学情宽表CDC 消费专用).
使用 ReplacingMergeTree 语义:按 ORDER BY 字段去重,保留 last_updated 最大版本。
返回 True 表示成功False 表示降级模式或写入失败。
"""
client = get_client()
if client is None:
return False
try:
client.insert(
"student_dashboard_view",
[
[
student_id,
class_id,
exam_id,
subject_id,
score,
rank_in_class,
knowledge_point_id,
mastery_level,
error_count,
last_updated,
]
],
column_names=[
"student_id",
"class_id",
"exam_id",
"subject_id",
"score",
"rank_in_class",
"knowledge_point_id",
"mastery_level",
"error_count",
"last_updated",
],
)
logger.info(
"student_dashboard_upserted",
student_id=student_id,
class_id=class_id,
exam_id=exam_id,
score=score,
)
return True
except Exception as exc: # noqa: BLE001
logger.warning(
"student_dashboard_upsert_failed_degraded",
error=str(exc),
student_id=student_id,
exam_id=exam_id,
)
return False
async def upsert_student_error(
student_id: str,
question_id: str,
knowledge_point_id: str,
error_count: int,
last_error_time: datetime,
content: str,
) -> bool:
"""写入/更新学生错题本CDC 消费专用).
返回 True 表示成功False 表示降级模式或写入失败。
"""
client = get_client()
if client is None:
return False
try:
client.insert(
"student_errors",
[
[
student_id,
question_id,
knowledge_point_id,
error_count,
last_error_time,
content,
]
],
column_names=[
"student_id",
"question_id",
"knowledge_point_id",
"error_count",
"last_error_time",
"content",
],
)
logger.info(
"student_error_upserted",
student_id=student_id,
question_id=question_id,
error_count=error_count,
)
return True
except Exception as exc: # noqa: BLE001
logger.warning(
"student_error_upsert_failed_degraded",
error=str(exc),
student_id=student_id,
question_id=question_id,
)
return False