feat(data-ana): implement complete CDC pipeline MySQL to ClickHouse
Debezium + Kafka + aiokafka consumer with table routing. E2E verified: MySQL INSERT to ClickHouse upsert.
This commit is contained in:
@@ -5,6 +5,7 @@
|
||||
保证服务在 ClickHouse 不可用时仍可启动并响应骨架数据。
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
@@ -216,3 +217,127 @@ async def ping() -> bool:
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("clickhouse_ping_failed", error=str(exc))
|
||||
return False
|
||||
|
||||
|
||||
async def upsert_student_dashboard(
|
||||
student_id: str,
|
||||
class_id: str,
|
||||
exam_id: str,
|
||||
subject_id: str,
|
||||
score: float,
|
||||
rank_in_class: int,
|
||||
knowledge_point_id: str,
|
||||
mastery_level: float,
|
||||
error_count: int,
|
||||
last_updated: datetime,
|
||||
) -> bool:
|
||||
"""写入/更新学生学情宽表(CDC 消费专用).
|
||||
|
||||
使用 ReplacingMergeTree 语义:按 ORDER BY 字段去重,保留 last_updated 最大版本。
|
||||
返回 True 表示成功;False 表示降级模式或写入失败。
|
||||
"""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
client.insert(
|
||||
"student_dashboard_view",
|
||||
[
|
||||
[
|
||||
student_id,
|
||||
class_id,
|
||||
exam_id,
|
||||
subject_id,
|
||||
score,
|
||||
rank_in_class,
|
||||
knowledge_point_id,
|
||||
mastery_level,
|
||||
error_count,
|
||||
last_updated,
|
||||
]
|
||||
],
|
||||
column_names=[
|
||||
"student_id",
|
||||
"class_id",
|
||||
"exam_id",
|
||||
"subject_id",
|
||||
"score",
|
||||
"rank_in_class",
|
||||
"knowledge_point_id",
|
||||
"mastery_level",
|
||||
"error_count",
|
||||
"last_updated",
|
||||
],
|
||||
)
|
||||
logger.info(
|
||||
"student_dashboard_upserted",
|
||||
student_id=student_id,
|
||||
class_id=class_id,
|
||||
exam_id=exam_id,
|
||||
score=score,
|
||||
)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"student_dashboard_upsert_failed_degraded",
|
||||
error=str(exc),
|
||||
student_id=student_id,
|
||||
exam_id=exam_id,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def upsert_student_error(
|
||||
student_id: str,
|
||||
question_id: str,
|
||||
knowledge_point_id: str,
|
||||
error_count: int,
|
||||
last_error_time: datetime,
|
||||
content: str,
|
||||
) -> bool:
|
||||
"""写入/更新学生错题本(CDC 消费专用).
|
||||
|
||||
返回 True 表示成功;False 表示降级模式或写入失败。
|
||||
"""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
client.insert(
|
||||
"student_errors",
|
||||
[
|
||||
[
|
||||
student_id,
|
||||
question_id,
|
||||
knowledge_point_id,
|
||||
error_count,
|
||||
last_error_time,
|
||||
content,
|
||||
]
|
||||
],
|
||||
column_names=[
|
||||
"student_id",
|
||||
"question_id",
|
||||
"knowledge_point_id",
|
||||
"error_count",
|
||||
"last_error_time",
|
||||
"content",
|
||||
],
|
||||
)
|
||||
logger.info(
|
||||
"student_error_upserted",
|
||||
student_id=student_id,
|
||||
question_id=question_id,
|
||||
error_count=error_count,
|
||||
)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"student_error_upsert_failed_degraded",
|
||||
error=str(exc),
|
||||
student_id=student_id,
|
||||
question_id=question_id,
|
||||
)
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user