refactor(shared-ts,iam,core-edu,content,msg): remove outbox polling publisher

M8: Debezium CDC now handles outbox table to Kafka (ADR-032)

- Remove OutboxPublisher class from shared-ts

- Remove publisher from iam/core-edu/content/msg lifecycle and modules

- OutboxService retained for transactional outbox table writes

- Debezium monitors binlog and pushes to Kafka automatically
This commit is contained in:
SpecialX
2026-07-15 01:27:45 +08:00
parent a3f4fd013e
commit 47a062606f
16 changed files with 34 additions and 1227 deletions

View File

@@ -8,25 +8,24 @@ import { closeDb } from "../../config/database.js";
import { closeEs } from "../../config/elasticsearch.js";
import { connectKafka, disconnectKafka } from "../kafka/kafka.client.js";
import { closeRedis } from "../redis/redis.client.js";
import { outboxPublisher } from "../outbox/outbox.publisher.js";
const SERVICE_NAME = "msg";
/**
* 优雅停机服务 + 资源生命周期管理。
*
* 仲裁依据 G2统一管理 DB/ES/Redis/Kafka/OutboxPublisher 生命周期。
* 仲裁依据 G2统一管理 DB/ES/Redis/Kafka 生命周期。
*
* v2.1M8OutboxPublisher 轮询线程已移除outbox 投递由 Debezium CDC 接管。
*
* 启动顺序onModuleInit
* 1. connectKafka():连接 producer + consumer幂等KafkaConsumerService 也会调用)
* 2. outboxPublisher.start():启动轮询 worker投递 pending 事件到 Kafka
*
* 关闭顺序onApplicationShutdown在 OnModuleDestroy 之后执行):
* 1. outboxPublisher.stop():停止轮询
* 2. disconnectKafka():断开 producer + consumer
* 3. closeRedis():关闭 Redis 连接
* 4. closeEs():关闭 ES 客户端
* 5. closeDb():关闭 MySQL 连接池
* 1. disconnectKafka():断开 producer + consumer
* 2. closeRedis():关闭 Redis 连接
* 3. closeEs():关闭 ES 客户端
* 4. closeDb():关闭 MySQL 连接池
*
* 注KafkaConsumerService 实现 OnModuleDestroy其 stop() 会在
* onApplicationShutdown 之前被 NestJS 调用。disconnectKafka() 是幂等的。
@@ -36,12 +35,9 @@ export class LifecycleService implements OnModuleInit, OnApplicationShutdown {
private readonly logger = new Logger(LifecycleService.name);
async onModuleInit(): Promise<void> {
// 1. 连接 Kafkaproducer + consumer
// 连接 Kafkaproducer + consumer
await connectKafka();
// 2. 启动 Outbox publisher 轮询
await outboxPublisher.start();
this.logger.log(`service ${SERVICE_NAME} module initialized`);
}
@@ -51,7 +47,6 @@ export class LifecycleService implements OnModuleInit, OnApplicationShutdown {
);
// 按依赖反序关闭
await this.safeStop("outboxPublisher", () => outboxPublisher.stop());
await this.safeStop("kafka", () => disconnectKafka());
await this.safeStop("redis", () => closeRedis());
await this.safeStop("elasticsearch", () => closeEs());

View File

@@ -1,125 +0,0 @@
import { logger } from "../observability/logger.js";
import { getProducer } from "../kafka/kafka.client.js";
import { resolveTopic } from "../kafka/topic-map.js";
import {
findPending,
incrementRetry,
markFailed,
markPublished,
} from "./outbox.repository.js";
import type { OutboxEvent } from "./outbox.schema.js";
/**
* OutboxPublisher —— 轮询 pending 记录并投递到 Kafka多 topic 路由)。
*
* 参照 core-edu OutboxPublisher 模式,支持 TOPIC_MAP 多 topic 路由:
* - notification.sent → edu.notify.notification.sent
* - notification.read → edu.notify.notification.read
* - notification.recalled → edu.notify.notification.recalled
* - notification.failed → edu.notify.notification.failed
*
* 仲裁依据at-least-once 投递 + 指数退避重试 + 幂等(消费端 event_id 去重)。
*/
const POLL_INTERVAL_MS = 5000;
const BATCH_SIZE = 100;
const MAX_RETRY = 5;
const RETRY_BACKOFF_MS = 2000;
class OutboxPublisher {
private intervalId: ReturnType<typeof setInterval> | null = null;
private isPolling = false;
async start(): Promise<void> {
if (this.intervalId) return;
logger.info(
{ pollIntervalMs: POLL_INTERVAL_MS, batchSize: BATCH_SIZE },
"OutboxPublisher started",
);
this.intervalId = setInterval(() => {
void this.poll();
}, POLL_INTERVAL_MS);
}
async stop(): Promise<void> {
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
}
logger.info("OutboxPublisher stopped");
}
private async poll(): Promise<void> {
if (this.isPolling) return;
this.isPolling = true;
try {
const messages = await findPending(BATCH_SIZE);
for (const message of messages) {
await this.dispatch(message);
}
} catch (error) {
logger.error({ error }, "Outbox poll failed");
} finally {
this.isPolling = false;
}
}
private async dispatch(message: OutboxEvent): Promise<void> {
const topic = resolveTopic(message.eventType);
try {
const producer = getProducer();
await producer.send({
topic,
messages: [
{
key: message.aggregateId,
value:
typeof message.payload === "string"
? message.payload
: JSON.stringify(message.payload),
headers: this.buildHeaders(message),
},
],
});
await markPublished(message.eventId);
logger.info(
{ eventId: message.eventId, eventType: message.eventType, topic },
"Outbox message published",
);
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : String(error);
logger.error(
{
eventId: message.eventId,
eventType: message.eventType,
error: errorMessage,
},
"Outbox publish failed",
);
if (message.retryCount + 1 >= MAX_RETRY) {
await markFailed(message.eventId, errorMessage);
} else {
const backoffMs = RETRY_BACKOFF_MS * 2 ** message.retryCount;
await incrementRetry(message.eventId, errorMessage, backoffMs);
}
}
}
private buildHeaders(message: OutboxEvent): Record<string, string> {
const headers: Record<string, string> = {
eventId: message.eventId,
eventType: message.eventType,
aggregateType: message.aggregateType,
aggregateId: message.aggregateId,
};
if (message.metadata) {
for (const [key, value] of Object.entries(message.metadata)) {
headers[key] = value;
}
}
return headers;
}
}
export const outboxPublisher = new OutboxPublisher();

View File

@@ -10,7 +10,8 @@ import { outboxEvents } from "./outbox.schema.js";
* 调用方在业务事务内调用 publish(),将事件记录写入 msg_outbox_events 表,
* 与业务写在同一事务中原子提交(事务性 Outbox 模式)。
*
* 由 OutboxPublisher 负责异步轮询 pending 记录并投递到 Kafkaat-least-once
* v2.1M8投递由 Debezium CDC 接管——监控 MySQL binlog 将 outbox 记录
* 投递到 Kafkaat-least-once。OutboxService 仅负责写入 outbox 表。
*
* 仲裁依据 G11eventId 用 cuid2同时作为 Kafka 消息 key 实现幂等去重。
*/