feat(data-ana): implement complete CDC pipeline MySQL to ClickHouse

Debezium + Kafka + aiokafka consumer with table routing.

E2E verified: MySQL INSERT to ClickHouse upsert.
This commit is contained in:
SpecialX
2026-07-09 13:02:59 +08:00
parent 958b17c9d8
commit 1f901c5b20
10 changed files with 595 additions and 43 deletions

View File

@@ -0,0 +1,14 @@
<clickhouse>
<!-- 覆盖 default-user.xml 的本地限制,允许 default 用户从任意 IP 用密码访问 -->
<users>
<default>
<password>clickhouse</password>
<networks>
<ip>::/0</ip>
</networks>
<profile>default</profile>
<quota>default</quota>
<access_management>1</access_management>
</default>
</users>
</clickhouse>

View File

@@ -44,7 +44,13 @@ services:
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
# 双 listenerINSIDE 容器间互访kafka:29092OUTSIDE 主机访问(localhost:9092
# 必须用 INSIDE 作为 inter.broker.listener.name否则 Debezium Connect 拿到 metadata
# 后会切回 advertised.listeners 中的 localhost导致连接失败
KAFKA_LISTENERS: INSIDE://:29092,OUTSIDE://:9092
KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:29092,OUTSIDE://localhost:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
ports:
@@ -76,6 +82,8 @@ services:
- "9000:9000"
volumes:
- clickhouse_data:/var/lib/clickhouse
# 覆盖默认 default-user.xml 限制(默认仅允许 127.0.0.1/::1 无密码访问)
- ./clickhouse/users.d/custom-users.xml:/etc/clickhouse-server/users.d/custom-users.xml:ro
healthcheck:
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
interval: 10s
@@ -127,6 +135,50 @@ services:
ports:
- "16686:16686"
- "4318:4318"
# ============================================================
# Debezium Connect - CDC 链路核心
# 监听 MySQL binlog → 写入 Kafka topic
# topic 命名约定:<prefix>.<database>.<table>(如 edu-cdc.next_edu_cloud.grades
# ============================================================
debezium-connect:
image: quay.io/debezium/connect:2.7
container_name: edu-debezium
profiles: ["p4", "p5", "p6"]
restart: unless-stopped
depends_on:
kafka:
condition: service_healthy
environment:
# Kafka Connect 基础配置Debezium 2.x 容器映射规则:环境变量名大写 → connect 配置项)
# 必须用 INSIDE listener (kafka:29092),否则会拿到 OUTSIDE 的 localhost metadata 导致连不上
BOOTSTRAP_SERVERS: kafka:29092
GROUP_ID: edu-debezium
CONFIG_STORAGE_TOPIC: edu-connect-configs
OFFSET_STORAGE_TOPIC: edu-connect-offsets
STATUS_STORAGE_TOPIC: edu-connect-status
# 内部 converter 配置(必须与 Debezium 事件格式一致)
CONFIG_STORAGE_REPLICATION_FACTOR: "1"
OFFSET_STORAGE_REPLICATION_FACTOR: "1"
STATUS_STORAGE_REPLICATION_FACTOR: "1"
KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
KEY_CONVERTER_SCHEMAS_ENABLE: "false"
VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
# 监听端口
REST_PORT: 8083
REST_ADVERTISED_HOST_NAME: debezium-connect
# 日志级别
LOG_LEVEL: INFO
ports:
- "8083:8083"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8083/connectors"]
interval: 15s
timeout: 5s
start_period: 30s
retries: 10
networks:
- default
prometheus:
image: docker.m.daocloud.io/prom/prometheus:v2.51.0
container_name: edu-prometheus