feat(data-ana): implement complete CDC pipeline MySQL to ClickHouse

Debezium + Kafka + aiokafka consumer with table routing.

E2E verified: MySQL INSERT to ClickHouse upsert.
This commit is contained in:
SpecialX
2026-07-09 13:02:59 +08:00
parent 958b17c9d8
commit 1f901c5b20
10 changed files with 595 additions and 43 deletions

View File

@@ -8,6 +8,9 @@ class Settings(BaseSettings):
ClickHouse 连接参数为可选:当 clickhouse_host 为空字符串时,
服务进入降级模式(查询方法返回 None / 空数据),保证服务可启动。
Kafka 连接参数为可选:当 kafka_brokers 为空字符串时,
CDC 消费者不启动(降级模式),保证服务可启动。
"""
port: int = 3006
@@ -22,8 +25,20 @@ class Settings(BaseSettings):
log_level: str = "info"
# 开发模式开关("true"/"false"
dev_mode: str = "false"
# Kafka brokersCDC 消费预留,暂不实现
kafka_brokers: str = "localhost:9092"
# Kafka brokersCDC 消费;留空则不启动消费者
# 主机访问用 localhost:9092容器内访问用 kafka:29092
kafka_brokers: str = ""
# CDC 消费组 id
kafka_group_id: str = "data-ana-cdc-consumer"
# 要消费的 CDC topicDebezium 默认命名:<prefix>.<database>.<table>
# 用逗号分隔多个 topic
kafka_cdc_topics: str = (
"edu-cdc.next_edu_cloud.core_edu_grades,"
"edu-cdc.next_edu_cloud.core_edu_exams,"
"edu-cdc.next_edu_cloud.classes"
)
# 消费者自动偏移重置策略earliest / latest
kafka_auto_offset_reset: str = "earliest"
model_config = {"env_file": ".env", "env_prefix": ""}