feat(infra): proto-to-graphql generator + debezium outbox connector
- M0: parse proto to generate Federation 2 subgraph SDL - M0.5: Debezium Connect monitors 4 outbox tables - docker-compose: extend Debezium profiles to default+p3+p4+p5+p6 - package.json: add proto:gen-graphql script
This commit is contained in:
37
infra/debezium/connectors/edu-outbox-connector.json
Normal file
37
infra/debezium/connectors/edu-outbox-connector.json
Normal file
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"name": "edu-outbox-connector",
|
||||
"config": {
|
||||
"connector.class": "io.debezium.connector.mysql.MySqlConnector",
|
||||
"database.hostname": "MYSQL_HOST_PLACEHOLDER",
|
||||
"database.port": "MYSQL_PORT_PLACEHOLDER",
|
||||
"database.user": "MYSQL_USER_PLACEHOLDER",
|
||||
"database.password": "MYSQL_PASSWORD_PLACEHOLDER",
|
||||
"database.server.id": "5400",
|
||||
"database.allowPublicKeyRetrieval": "true",
|
||||
"database.include.list": "next_edu_cloud",
|
||||
"table.include.list": "next_edu_cloud.iam_outbox,next_edu_cloud.core_edu_outbox,next_edu_cloud.content_outbox_events,next_edu_cloud.msg_outbox_events",
|
||||
"database.server.name": "edu-cdc",
|
||||
"topic.prefix": "edu-cdc",
|
||||
"schema.history.internal.kafka.bootstrap.servers": "kafka:29092",
|
||||
"schema.history.internal.kafka.topic": "edu-cdc-schema-history",
|
||||
|
||||
"transforms": "unwrap,topicRouter",
|
||||
"transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
|
||||
"transforms.unwrap.drop.tombstones": "true",
|
||||
"transforms.unwrap.delete.handling.mode": "rewrite",
|
||||
|
||||
"transforms.topicRouter.type": "org.apache.kafka.connect.transforms.RegexRouter",
|
||||
"transforms.topicRouter.regex": "edu-cdc\\.next_edu_cloud\\.(iam_outbox|core_edu_outbox|content_outbox_events|msg_outbox_events)",
|
||||
"transforms.topicRouter.replacement": "edu.outbox.events",
|
||||
|
||||
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
|
||||
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
|
||||
"value.converter.schemas.enable": "false",
|
||||
|
||||
"snapshot.mode": "schema_only_recovery",
|
||||
"tombstones.on.delete": "false",
|
||||
"poll.interval.ms": "500",
|
||||
"max.batch.size": "2048",
|
||||
"max.queue.size": "8192"
|
||||
}
|
||||
}
|
||||
98
infra/debezium/register-outbox-connectors.sh
Normal file
98
infra/debezium/register-outbox-connectors.sh
Normal file
@@ -0,0 +1,98 @@
|
||||
#!/bin/bash
|
||||
# Debezium Outbox Connectors 注册脚本(v2.1 M0.5)
|
||||
#
|
||||
# 功能:
|
||||
# - 向 Debezium Connect 注册 outbox 表的 CDC connector
|
||||
# - 监听 next_edu_cloud 数据库的 4 张 outbox 表
|
||||
# - 使用 ExtractNewRecordState SMT 简化事件格式
|
||||
# - 使用 TopicRouter SMT 根据 event_type 字段路由到业务 topic
|
||||
#
|
||||
# 使用:
|
||||
# bash infra/debezium/register-outbox-connectors.sh
|
||||
#
|
||||
# 前置条件:
|
||||
# - Debezium Connect 已启动(端口 8083)
|
||||
# - MySQL binlog 已启用(row mode)
|
||||
# - outbox 表已存在
|
||||
#
|
||||
# v2.1 设计(见 §5.4 / §6.1 / ADR-032):
|
||||
# - 业务代码只写业务表 + Outbox 表(事务内原子)
|
||||
# - Debezium 监听 binlog 自动投递(Transaction Log Tailing)
|
||||
# - 废弃 OutboxPublisher 轮询线程
|
||||
# - 保证 at-least-once 语义(Debezium offset 管理)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DEBEZIUM_HOST="${DEBEZIUM_HOST:-localhost}"
|
||||
DEBEZIUM_PORT="${DEBEZIUM_PORT:-8083}"
|
||||
DEBEZIUM_URL="http://${DEBEZIUM_HOST}:${DEBEZIUM_PORT}"
|
||||
|
||||
MYSQL_HOST="${MYSQL_HOST:-edu-mysql}"
|
||||
MYSQL_PORT="${MYSQL_PORT:-3306}"
|
||||
MYSQL_USER="${MYSQL_USER:-debezium}"
|
||||
MYSQL_PASSWORD="${MYSQL_PASSWORD:-debezium}"
|
||||
|
||||
CONNECTORS_DIR="$(dirname "$0")/connectors"
|
||||
|
||||
echo "=== Debezium Outbox Connector 注册(v2.1) ==="
|
||||
echo "目标:${DEBEZIUM_URL}"
|
||||
echo "MySQL:${MYSQL_HOST}:${MYSQL_PORT}"
|
||||
echo ""
|
||||
|
||||
# 等待 Debezium Connect 就绪
|
||||
echo "等待 Debezium Connect 就绪..."
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "${DEBEZIUM_URL}/connectors" > /dev/null 2>&1; then
|
||||
echo "✓ Debezium Connect 就绪"
|
||||
break
|
||||
fi
|
||||
echo " 尝试 ${i}/30..."
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# 注册所有 connector
|
||||
for connector_file in "${CONNECTORS_DIR}"/*.json; do
|
||||
if [ ! -f "$connector_file" ]; then
|
||||
echo "⚠ 无 connector 配置文件"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
connector_name=$(basename "$connector_file" .json)
|
||||
echo ""
|
||||
echo "注册 connector: ${connector_name}"
|
||||
|
||||
# 替换占位符为环境变量值
|
||||
config_payload=$(sed \
|
||||
-e "s|MYSQL_HOST_PLACEHOLDER|${MYSQL_HOST}|g" \
|
||||
-e "s|MYSQL_PORT_PLACEHOLDER|${MYSQL_PORT}|g" \
|
||||
-e "s|MYSQL_USER_PLACEHOLDER|${MYSQL_USER}|g" \
|
||||
-e "s|MYSQL_PASSWORD_PLACEHOLDER|${MYSQL_PASSWORD}|g" \
|
||||
"$connector_file")
|
||||
|
||||
# 检查是否已存在
|
||||
if curl -sf "${DEBEZIUM_URL}/connectors/${connector_name}" > /dev/null 2>&1; then
|
||||
echo " ⚠ 已存在,更新配置..."
|
||||
echo "$config_payload" | curl -sX PUT "${DEBEZIUM_URL}/connectors/${connector_name}/config" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @- | jq .
|
||||
else
|
||||
# 包装为创建请求
|
||||
payload=$(jq -n --arg name "$connector_name" --argjson config "$config_payload" \
|
||||
'{name: $name, config: $config}')
|
||||
echo "$payload" | curl -sX POST "${DEBEZIUM_URL}/connectors" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @- | jq .
|
||||
fi
|
||||
|
||||
echo " ✓ ${connector_name} 已注册"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== 注册完成 ==="
|
||||
echo ""
|
||||
echo "查看 connector 状态:"
|
||||
echo " curl ${DEBEZIUM_URL}/connectors"
|
||||
echo " curl ${DEBEZIUM_URL}/connectors/<name>/status"
|
||||
echo ""
|
||||
echo "删除 connector:"
|
||||
echo " curl -X DELETE ${DEBEZIUM_URL}/connectors/<name>"
|
||||
@@ -136,14 +136,16 @@ services:
|
||||
- "16686:16686"
|
||||
- "4318:4318"
|
||||
# ============================================================
|
||||
# Debezium Connect - CDC 链路核心
|
||||
# Debezium Connect - CDC 链路核心(v2.1:Outbox 表监听 + 自动投递)
|
||||
# 监听 MySQL binlog → 写入 Kafka topic
|
||||
# topic 命名约定:<prefix>.<database>.<table>(如 edu-cdc.next_edu_cloud.grades)
|
||||
# v2.1 设计(ADR-032):业务代码只写 outbox 表,Debezium 监听 binlog 自动投递
|
||||
# 废弃 OutboxPublisher 轮询线程(见 M0.5 / M8)
|
||||
# 注册脚本:bash infra/debezium/register-outbox-connectors.sh
|
||||
# ============================================================
|
||||
debezium-connect:
|
||||
image: quay.io/debezium/connect:2.7
|
||||
container_name: edu-debezium
|
||||
profiles: ["p4", "p5", "p6"]
|
||||
profiles: ["default", "p3", "p4", "p5", "p6"]
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
kafka:
|
||||
|
||||
Reference in New Issue
Block a user