feat(data-ana): implement complete CDC pipeline MySQL to ClickHouse
Debezium + Kafka + aiokafka consumer with table routing. E2E verified: MySQL INSERT to ClickHouse upsert.
This commit is contained in:
14
infra/clickhouse/users.d/custom-users.xml
Normal file
14
infra/clickhouse/users.d/custom-users.xml
Normal file
@@ -0,0 +1,14 @@
|
||||
<clickhouse>
|
||||
<!-- 覆盖 default-user.xml 的本地限制,允许 default 用户从任意 IP 用密码访问 -->
|
||||
<users>
|
||||
<default>
|
||||
<password>clickhouse</password>
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
<profile>default</profile>
|
||||
<quota>default</quota>
|
||||
<access_management>1</access_management>
|
||||
</default>
|
||||
</users>
|
||||
</clickhouse>
|
||||
@@ -44,7 +44,13 @@ services:
|
||||
environment:
|
||||
KAFKA_BROKER_ID: 1
|
||||
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
||||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
|
||||
# 双 listener:INSIDE 容器间互访(kafka:29092),OUTSIDE 主机访问(localhost:9092)
|
||||
# 必须用 INSIDE 作为 inter.broker.listener.name,否则 Debezium Connect 拿到 metadata
|
||||
# 后会切回 advertised.listeners 中的 localhost,导致连接失败
|
||||
KAFKA_LISTENERS: INSIDE://:29092,OUTSIDE://:9092
|
||||
KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:29092,OUTSIDE://localhost:9092
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
|
||||
KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
|
||||
ports:
|
||||
@@ -76,6 +82,8 @@ services:
|
||||
- "9000:9000"
|
||||
volumes:
|
||||
- clickhouse_data:/var/lib/clickhouse
|
||||
# 覆盖默认 default-user.xml 限制(默认仅允许 127.0.0.1/::1 无密码访问)
|
||||
- ./clickhouse/users.d/custom-users.xml:/etc/clickhouse-server/users.d/custom-users.xml:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
|
||||
interval: 10s
|
||||
@@ -127,6 +135,50 @@ services:
|
||||
ports:
|
||||
- "16686:16686"
|
||||
- "4318:4318"
|
||||
# ============================================================
|
||||
# Debezium Connect - CDC 链路核心
|
||||
# 监听 MySQL binlog → 写入 Kafka topic
|
||||
# topic 命名约定:<prefix>.<database>.<table>(如 edu-cdc.next_edu_cloud.grades)
|
||||
# ============================================================
|
||||
debezium-connect:
|
||||
image: quay.io/debezium/connect:2.7
|
||||
container_name: edu-debezium
|
||||
profiles: ["p4", "p5", "p6"]
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
kafka:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
# Kafka Connect 基础配置(Debezium 2.x 容器映射规则:环境变量名大写 → connect 配置项)
|
||||
# 必须用 INSIDE listener (kafka:29092),否则会拿到 OUTSIDE 的 localhost metadata 导致连不上
|
||||
BOOTSTRAP_SERVERS: kafka:29092
|
||||
GROUP_ID: edu-debezium
|
||||
CONFIG_STORAGE_TOPIC: edu-connect-configs
|
||||
OFFSET_STORAGE_TOPIC: edu-connect-offsets
|
||||
STATUS_STORAGE_TOPIC: edu-connect-status
|
||||
# 内部 converter 配置(必须与 Debezium 事件格式一致)
|
||||
CONFIG_STORAGE_REPLICATION_FACTOR: "1"
|
||||
OFFSET_STORAGE_REPLICATION_FACTOR: "1"
|
||||
STATUS_STORAGE_REPLICATION_FACTOR: "1"
|
||||
KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||
VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||
KEY_CONVERTER_SCHEMAS_ENABLE: "false"
|
||||
VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
|
||||
# 监听端口
|
||||
REST_PORT: 8083
|
||||
REST_ADVERTISED_HOST_NAME: debezium-connect
|
||||
# 日志级别
|
||||
LOG_LEVEL: INFO
|
||||
ports:
|
||||
- "8083:8083"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8083/connectors"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
start_period: 30s
|
||||
retries: 10
|
||||
networks:
|
||||
- default
|
||||
prometheus:
|
||||
image: docker.m.daocloud.io/prom/prometheus:v2.51.0
|
||||
container_name: edu-prometheus
|
||||
|
||||
Reference in New Issue
Block a user