refactor(shared-ts,iam,core-edu,content,msg): remove outbox polling publisher

M8: Debezium CDC now handles outbox table to Kafka (ADR-032)

- Remove OutboxPublisher class from shared-ts

- Remove publisher from iam/core-edu/content/msg lifecycle and modules

- OutboxService retained for transactional outbox table writes

- Debezium monitors binlog and pushes to Kafka automatically
This commit is contained in:
SpecialX
2026-07-15 01:27:45 +08:00
parent a3f4fd013e
commit 47a062606f
16 changed files with 34 additions and 1227 deletions

View File

@@ -12,7 +12,6 @@ import { closeNeo4j } from "./config/neo4j.js";
import { connectKafka, disconnectKafka } from "./config/kafka.js";
import { logger } from "./shared/observability/logger.js";
import { metricsRegistry } from "./shared/observability/metrics.js";
import { outboxPublisher } from "./shared/outbox/outbox.publisher.js";
import { neo4jSyncWorker } from "./shared/sync/neo4j-sync.worker.js";
import { esSyncWorker } from "./shared/sync/es-sync.worker.js";
import { ensureQuestionIndex, closeEs } from "./config/elasticsearch.js";
@@ -86,9 +85,6 @@ async function bootstrap(): Promise<void> {
"Content service started (HTTP + gRPC)",
);
// 启动 Outbox Publisher轮询 pending 事件投递 Kafka
await outboxPublisher.start();
// 创建/校验 ES 索引幂等ES 不可用时跳过)
await ensureQuestionIndex();
@@ -102,7 +98,6 @@ async function bootstrap(): Promise<void> {
logger.info("SIGTERM received, shutting down gracefully...");
await esSyncWorker.stop();
await neo4jSyncWorker.stop();
await outboxPublisher.stop();
await disconnectKafka();
await app.close();
await closeEs();
@@ -116,7 +111,6 @@ async function bootstrap(): Promise<void> {
logger.info("SIGINT received, shutting down gracefully...");
await esSyncWorker.stop();
await neo4jSyncWorker.stop();
await outboxPublisher.stop();
await disconnectKafka();
await app.close();
await closeEs();

View File

@@ -1,163 +0,0 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
const mockProducer = vi.hoisted(() => ({ send: vi.fn() }));
vi.mock("../../config/kafka.js", () => ({
producer: mockProducer,
}));
vi.mock("../observability/logger.js", () => ({
logger: {
info: vi.fn(),
error: vi.fn(),
warn: vi.fn(),
debug: vi.fn(),
},
}));
vi.mock("./outbox.repository.js", () => ({
outboxRepository: {
findPending: vi.fn(),
markPublished: vi.fn(),
incrementRetry: vi.fn(),
markFailed: vi.fn(),
create: vi.fn(),
},
}));
import { OutboxPublisher } from "./outbox.publisher.js";
import { outboxRepository } from "./outbox.repository.js";
import type { OutboxMessage } from "./outbox.schema.js";
function createMessage(overrides: Partial<OutboxMessage> = {}): OutboxMessage {
return {
id: "msg-1",
aggregateType: "Chapter",
aggregateId: "ch-1",
eventType: "chapter.created",
topic: "edu.content.chapter.events",
payload: '{"event_id":"e1"}',
status: "pending",
retryCount: 0,
createdAt: new Date(),
publishedAt: null,
nextRetryAt: null,
lastError: null,
...overrides,
};
}
describe("OutboxPublisher", () => {
let publisher: OutboxPublisher;
beforeEach(() => {
vi.clearAllMocks();
publisher = new OutboxPublisher();
});
describe("dispatch (via poll)", () => {
it("should send message to kafka and mark as published", async () => {
const message = createMessage();
vi.mocked(outboxRepository.findPending).mockResolvedValue([message]);
mockProducer.send.mockResolvedValue(undefined);
// Access private poll via casting
await (publisher as unknown as { poll: () => Promise<void> }).poll();
expect(mockProducer.send).toHaveBeenCalledWith({
topic: message.topic,
messages: [
{
key: message.aggregateId,
value: message.payload,
headers: {
eventId: message.id,
eventType: message.eventType,
aggregateType: message.aggregateType,
},
},
],
});
expect(outboxRepository.markPublished).toHaveBeenCalledWith(message.id);
});
it("should increment retry when send fails and below max retries", async () => {
const message = createMessage({ id: "msg-2", retryCount: 0 });
vi.mocked(outboxRepository.findPending).mockResolvedValue([message]);
mockProducer.send.mockRejectedValue(new Error("kafka down"));
await (publisher as unknown as { poll: () => Promise<void> }).poll();
expect(outboxRepository.incrementRetry).toHaveBeenCalledWith(
"msg-2",
"kafka down",
);
expect(outboxRepository.markFailed).not.toHaveBeenCalled();
});
it("should mark failed when retry count reaches max", async () => {
const message = createMessage({ id: "msg-3", retryCount: 4 });
vi.mocked(outboxRepository.findPending).mockResolvedValue([message]);
mockProducer.send.mockRejectedValue(new Error("kafka down"));
await (publisher as unknown as { poll: () => Promise<void> }).poll();
expect(outboxRepository.markFailed).toHaveBeenCalledWith(
"msg-3",
"kafka down",
);
expect(outboxRepository.incrementRetry).not.toHaveBeenCalled();
});
it("should handle non-Error rejection in dispatch", async () => {
const message = createMessage({ id: "msg-4", retryCount: 0 });
vi.mocked(outboxRepository.findPending).mockResolvedValue([message]);
mockProducer.send.mockRejectedValue("string error");
await (publisher as unknown as { poll: () => Promise<void> }).poll();
expect(outboxRepository.incrementRetry).toHaveBeenCalledWith(
"msg-4",
"string error",
);
});
});
describe("poll error handling", () => {
it("should swallow errors from findPending", async () => {
vi.mocked(outboxRepository.findPending).mockRejectedValue(
new Error("db error"),
);
await expect(
(publisher as unknown as { poll: () => Promise<void> }).poll(),
).resolves.not.toThrow();
});
it("should process multiple messages in a batch", async () => {
const messages = [
createMessage({ id: "m1" }),
createMessage({ id: "m2" }),
];
vi.mocked(outboxRepository.findPending).mockResolvedValue(messages);
mockProducer.send.mockResolvedValue(undefined);
await (publisher as unknown as { poll: () => Promise<void> }).poll();
expect(mockProducer.send).toHaveBeenCalledTimes(2);
expect(outboxRepository.markPublished).toHaveBeenCalledWith("m1");
expect(outboxRepository.markPublished).toHaveBeenCalledWith("m2");
});
});
describe("start/stop", () => {
it("should start and stop without error", async () => {
await publisher.start();
await publisher.stop();
// Should not throw
});
it("should stop without error when not started", async () => {
await publisher.stop();
});
});
});

View File

@@ -1,81 +0,0 @@
import { logger } from "../observability/logger.js";
import { producer } from "../../config/kafka.js";
import { outboxRepository } from "./outbox.repository.js";
import type { OutboxMessage } from "./outbox.schema.js";
const POLL_INTERVAL_MS = 5000;
const BATCH_SIZE = 100;
const MAX_RETRY = 5;
export class OutboxPublisher {
private intervalId: ReturnType<typeof setInterval> | null = null;
private isPolling = false;
async start(): Promise<void> {
logger.info("OutboxPublisher started");
this.intervalId = setInterval(() => {
void this.poll();
}, POLL_INTERVAL_MS);
}
async stop(): Promise<void> {
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
}
logger.info("OutboxPublisher stopped");
}
private async poll(): Promise<void> {
if (this.isPolling) return;
this.isPolling = true;
try {
const messages = await outboxRepository.findPending(BATCH_SIZE);
for (const message of messages) {
await this.dispatch(message);
}
} catch (error) {
logger.error({ error }, "Outbox poll failed");
} finally {
this.isPolling = false;
}
}
private async dispatch(message: OutboxMessage): Promise<void> {
try {
await producer.send({
topic: message.topic,
messages: [
{
key: message.aggregateId,
value: message.payload,
headers: {
eventId: message.id,
eventType: message.eventType,
aggregateType: message.aggregateType,
},
},
],
});
await outboxRepository.markPublished(message.id);
logger.info(
{ id: message.id, eventType: message.eventType, topic: message.topic },
"Outbox message published",
);
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : String(error);
logger.error(
{ error, id: message.id, eventType: message.eventType },
"Outbox publish failed",
);
if (message.retryCount + 1 >= MAX_RETRY) {
await outboxRepository.markFailed(message.id, errorMessage);
} else {
await outboxRepository.incrementRetry(message.id, errorMessage);
}
}
}
}
export const outboxPublisher = new OutboxPublisher();