refactor(shared-ts,iam,core-edu,content,msg): remove outbox polling publisher
M8: Debezium CDC now handles outbox table to Kafka (ADR-032) - Remove OutboxPublisher class from shared-ts - Remove publisher from iam/core-edu/content/msg lifecycle and modules - OutboxService retained for transactional outbox table writes - Debezium monitors binlog and pushes to Kafka automatically
This commit is contained in:
@@ -8,25 +8,24 @@ import { closeDb } from "../../config/database.js";
|
||||
import { closeEs } from "../../config/elasticsearch.js";
|
||||
import { connectKafka, disconnectKafka } from "../kafka/kafka.client.js";
|
||||
import { closeRedis } from "../redis/redis.client.js";
|
||||
import { outboxPublisher } from "../outbox/outbox.publisher.js";
|
||||
|
||||
const SERVICE_NAME = "msg";
|
||||
|
||||
/**
|
||||
* 优雅停机服务 + 资源生命周期管理。
|
||||
*
|
||||
* 仲裁依据 G2:统一管理 DB/ES/Redis/Kafka/OutboxPublisher 生命周期。
|
||||
* 仲裁依据 G2:统一管理 DB/ES/Redis/Kafka 生命周期。
|
||||
*
|
||||
* v2.1(M8):OutboxPublisher 轮询线程已移除,outbox 投递由 Debezium CDC 接管。
|
||||
*
|
||||
* 启动顺序(onModuleInit):
|
||||
* 1. connectKafka():连接 producer + consumer(幂等,KafkaConsumerService 也会调用)
|
||||
* 2. outboxPublisher.start():启动轮询 worker,投递 pending 事件到 Kafka
|
||||
*
|
||||
* 关闭顺序(onApplicationShutdown,在 OnModuleDestroy 之后执行):
|
||||
* 1. outboxPublisher.stop():停止轮询
|
||||
* 2. disconnectKafka():断开 producer + consumer
|
||||
* 3. closeRedis():关闭 Redis 连接
|
||||
* 4. closeEs():关闭 ES 客户端
|
||||
* 5. closeDb():关闭 MySQL 连接池
|
||||
* 1. disconnectKafka():断开 producer + consumer
|
||||
* 2. closeRedis():关闭 Redis 连接
|
||||
* 3. closeEs():关闭 ES 客户端
|
||||
* 4. closeDb():关闭 MySQL 连接池
|
||||
*
|
||||
* 注:KafkaConsumerService 实现 OnModuleDestroy,其 stop() 会在
|
||||
* onApplicationShutdown 之前被 NestJS 调用。disconnectKafka() 是幂等的。
|
||||
@@ -36,12 +35,9 @@ export class LifecycleService implements OnModuleInit, OnApplicationShutdown {
|
||||
private readonly logger = new Logger(LifecycleService.name);
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
// 1. 连接 Kafka(producer + consumer)
|
||||
// 连接 Kafka(producer + consumer)
|
||||
await connectKafka();
|
||||
|
||||
// 2. 启动 Outbox publisher 轮询
|
||||
await outboxPublisher.start();
|
||||
|
||||
this.logger.log(`service ${SERVICE_NAME} module initialized`);
|
||||
}
|
||||
|
||||
@@ -51,7 +47,6 @@ export class LifecycleService implements OnModuleInit, OnApplicationShutdown {
|
||||
);
|
||||
|
||||
// 按依赖反序关闭
|
||||
await this.safeStop("outboxPublisher", () => outboxPublisher.stop());
|
||||
await this.safeStop("kafka", () => disconnectKafka());
|
||||
await this.safeStop("redis", () => closeRedis());
|
||||
await this.safeStop("elasticsearch", () => closeEs());
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
import { logger } from "../observability/logger.js";
|
||||
import { getProducer } from "../kafka/kafka.client.js";
|
||||
import { resolveTopic } from "../kafka/topic-map.js";
|
||||
import {
|
||||
findPending,
|
||||
incrementRetry,
|
||||
markFailed,
|
||||
markPublished,
|
||||
} from "./outbox.repository.js";
|
||||
import type { OutboxEvent } from "./outbox.schema.js";
|
||||
|
||||
/**
|
||||
* OutboxPublisher —— 轮询 pending 记录并投递到 Kafka(多 topic 路由)。
|
||||
*
|
||||
* 参照 core-edu OutboxPublisher 模式,支持 TOPIC_MAP 多 topic 路由:
|
||||
* - notification.sent → edu.notify.notification.sent
|
||||
* - notification.read → edu.notify.notification.read
|
||||
* - notification.recalled → edu.notify.notification.recalled
|
||||
* - notification.failed → edu.notify.notification.failed
|
||||
*
|
||||
* 仲裁依据:at-least-once 投递 + 指数退避重试 + 幂等(消费端 event_id 去重)。
|
||||
*/
|
||||
|
||||
const POLL_INTERVAL_MS = 5000;
|
||||
const BATCH_SIZE = 100;
|
||||
const MAX_RETRY = 5;
|
||||
const RETRY_BACKOFF_MS = 2000;
|
||||
|
||||
class OutboxPublisher {
|
||||
private intervalId: ReturnType<typeof setInterval> | null = null;
|
||||
private isPolling = false;
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.intervalId) return;
|
||||
logger.info(
|
||||
{ pollIntervalMs: POLL_INTERVAL_MS, batchSize: BATCH_SIZE },
|
||||
"OutboxPublisher started",
|
||||
);
|
||||
this.intervalId = setInterval(() => {
|
||||
void this.poll();
|
||||
}, POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
}
|
||||
logger.info("OutboxPublisher stopped");
|
||||
}
|
||||
|
||||
private async poll(): Promise<void> {
|
||||
if (this.isPolling) return;
|
||||
this.isPolling = true;
|
||||
try {
|
||||
const messages = await findPending(BATCH_SIZE);
|
||||
for (const message of messages) {
|
||||
await this.dispatch(message);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error({ error }, "Outbox poll failed");
|
||||
} finally {
|
||||
this.isPolling = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async dispatch(message: OutboxEvent): Promise<void> {
|
||||
const topic = resolveTopic(message.eventType);
|
||||
try {
|
||||
const producer = getProducer();
|
||||
await producer.send({
|
||||
topic,
|
||||
messages: [
|
||||
{
|
||||
key: message.aggregateId,
|
||||
value:
|
||||
typeof message.payload === "string"
|
||||
? message.payload
|
||||
: JSON.stringify(message.payload),
|
||||
headers: this.buildHeaders(message),
|
||||
},
|
||||
],
|
||||
});
|
||||
await markPublished(message.eventId);
|
||||
logger.info(
|
||||
{ eventId: message.eventId, eventType: message.eventType, topic },
|
||||
"Outbox message published",
|
||||
);
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
logger.error(
|
||||
{
|
||||
eventId: message.eventId,
|
||||
eventType: message.eventType,
|
||||
error: errorMessage,
|
||||
},
|
||||
"Outbox publish failed",
|
||||
);
|
||||
if (message.retryCount + 1 >= MAX_RETRY) {
|
||||
await markFailed(message.eventId, errorMessage);
|
||||
} else {
|
||||
const backoffMs = RETRY_BACKOFF_MS * 2 ** message.retryCount;
|
||||
await incrementRetry(message.eventId, errorMessage, backoffMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private buildHeaders(message: OutboxEvent): Record<string, string> {
|
||||
const headers: Record<string, string> = {
|
||||
eventId: message.eventId,
|
||||
eventType: message.eventType,
|
||||
aggregateType: message.aggregateType,
|
||||
aggregateId: message.aggregateId,
|
||||
};
|
||||
if (message.metadata) {
|
||||
for (const [key, value] of Object.entries(message.metadata)) {
|
||||
headers[key] = value;
|
||||
}
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
}
|
||||
|
||||
export const outboxPublisher = new OutboxPublisher();
|
||||
@@ -10,7 +10,8 @@ import { outboxEvents } from "./outbox.schema.js";
|
||||
* 调用方在业务事务内调用 publish(),将事件记录写入 msg_outbox_events 表,
|
||||
* 与业务写在同一事务中原子提交(事务性 Outbox 模式)。
|
||||
*
|
||||
* 由 OutboxPublisher 负责异步轮询 pending 记录并投递到 Kafka(at-least-once)。
|
||||
* v2.1(M8):投递由 Debezium CDC 接管——监控 MySQL binlog 将 outbox 记录
|
||||
* 投递到 Kafka(at-least-once)。OutboxService 仅负责写入 outbox 表。
|
||||
*
|
||||
* 仲裁依据 G11:eventId 用 cuid2,同时作为 Kafka 消息 key 实现幂等去重。
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user