feat(data-ana): implement complete CDC pipeline MySQL to ClickHouse
Debezium + Kafka + aiokafka consumer with table routing. E2E verified: MySQL INSERT to ClickHouse upsert.
This commit is contained in:
@@ -8,6 +8,9 @@ class Settings(BaseSettings):
|
||||
|
||||
ClickHouse 连接参数为可选:当 clickhouse_host 为空字符串时,
|
||||
服务进入降级模式(查询方法返回 None / 空数据),保证服务可启动。
|
||||
|
||||
Kafka 连接参数为可选:当 kafka_brokers 为空字符串时,
|
||||
CDC 消费者不启动(降级模式),保证服务可启动。
|
||||
"""
|
||||
|
||||
port: int = 3006
|
||||
@@ -22,8 +25,20 @@ class Settings(BaseSettings):
|
||||
log_level: str = "info"
|
||||
# 开发模式开关("true"/"false")
|
||||
dev_mode: str = "false"
|
||||
# Kafka brokers(CDC 消费预留,暂不实现)
|
||||
kafka_brokers: str = "localhost:9092"
|
||||
# Kafka brokers(CDC 消费;留空则不启动消费者)
|
||||
# 主机访问用 localhost:9092,容器内访问用 kafka:29092
|
||||
kafka_brokers: str = ""
|
||||
# CDC 消费组 id
|
||||
kafka_group_id: str = "data-ana-cdc-consumer"
|
||||
# 要消费的 CDC topic(Debezium 默认命名:<prefix>.<database>.<table>)
|
||||
# 用逗号分隔多个 topic
|
||||
kafka_cdc_topics: str = (
|
||||
"edu-cdc.next_edu_cloud.core_edu_grades,"
|
||||
"edu-cdc.next_edu_cloud.core_edu_exams,"
|
||||
"edu-cdc.next_edu_cloud.classes"
|
||||
)
|
||||
# 消费者自动偏移重置策略(earliest / latest)
|
||||
kafka_auto_offset_reset: str = "earliest"
|
||||
|
||||
model_config = {"env_file": ".env", "env_prefix": ""}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user