refactor(shared-ts,iam,core-edu,content,msg): remove outbox polling publisher
M8: Debezium CDC now handles outbox table to Kafka (ADR-032) - Remove OutboxPublisher class from shared-ts - Remove publisher from iam/core-edu/content/msg lifecycle and modules - OutboxService retained for transactional outbox table writes - Debezium monitors binlog and pushes to Kafka automatically
This commit is contained in:
@@ -12,7 +12,6 @@ import { closeNeo4j } from "./config/neo4j.js";
|
||||
import { connectKafka, disconnectKafka } from "./config/kafka.js";
|
||||
import { logger } from "./shared/observability/logger.js";
|
||||
import { metricsRegistry } from "./shared/observability/metrics.js";
|
||||
import { outboxPublisher } from "./shared/outbox/outbox.publisher.js";
|
||||
import { neo4jSyncWorker } from "./shared/sync/neo4j-sync.worker.js";
|
||||
import { esSyncWorker } from "./shared/sync/es-sync.worker.js";
|
||||
import { ensureQuestionIndex, closeEs } from "./config/elasticsearch.js";
|
||||
@@ -86,9 +85,6 @@ async function bootstrap(): Promise<void> {
|
||||
"Content service started (HTTP + gRPC)",
|
||||
);
|
||||
|
||||
// 启动 Outbox Publisher(轮询 pending 事件投递 Kafka)
|
||||
await outboxPublisher.start();
|
||||
|
||||
// 创建/校验 ES 索引(幂等;ES 不可用时跳过)
|
||||
await ensureQuestionIndex();
|
||||
|
||||
@@ -102,7 +98,6 @@ async function bootstrap(): Promise<void> {
|
||||
logger.info("SIGTERM received, shutting down gracefully...");
|
||||
await esSyncWorker.stop();
|
||||
await neo4jSyncWorker.stop();
|
||||
await outboxPublisher.stop();
|
||||
await disconnectKafka();
|
||||
await app.close();
|
||||
await closeEs();
|
||||
@@ -116,7 +111,6 @@ async function bootstrap(): Promise<void> {
|
||||
logger.info("SIGINT received, shutting down gracefully...");
|
||||
await esSyncWorker.stop();
|
||||
await neo4jSyncWorker.stop();
|
||||
await outboxPublisher.stop();
|
||||
await disconnectKafka();
|
||||
await app.close();
|
||||
await closeEs();
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
const mockProducer = vi.hoisted(() => ({ send: vi.fn() }));
|
||||
|
||||
vi.mock("../../config/kafka.js", () => ({
|
||||
producer: mockProducer,
|
||||
}));
|
||||
|
||||
vi.mock("../observability/logger.js", () => ({
|
||||
logger: {
|
||||
info: vi.fn(),
|
||||
error: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("./outbox.repository.js", () => ({
|
||||
outboxRepository: {
|
||||
findPending: vi.fn(),
|
||||
markPublished: vi.fn(),
|
||||
incrementRetry: vi.fn(),
|
||||
markFailed: vi.fn(),
|
||||
create: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
import { OutboxPublisher } from "./outbox.publisher.js";
|
||||
import { outboxRepository } from "./outbox.repository.js";
|
||||
import type { OutboxMessage } from "./outbox.schema.js";
|
||||
|
||||
function createMessage(overrides: Partial<OutboxMessage> = {}): OutboxMessage {
|
||||
return {
|
||||
id: "msg-1",
|
||||
aggregateType: "Chapter",
|
||||
aggregateId: "ch-1",
|
||||
eventType: "chapter.created",
|
||||
topic: "edu.content.chapter.events",
|
||||
payload: '{"event_id":"e1"}',
|
||||
status: "pending",
|
||||
retryCount: 0,
|
||||
createdAt: new Date(),
|
||||
publishedAt: null,
|
||||
nextRetryAt: null,
|
||||
lastError: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("OutboxPublisher", () => {
|
||||
let publisher: OutboxPublisher;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
publisher = new OutboxPublisher();
|
||||
});
|
||||
|
||||
describe("dispatch (via poll)", () => {
|
||||
it("should send message to kafka and mark as published", async () => {
|
||||
const message = createMessage();
|
||||
vi.mocked(outboxRepository.findPending).mockResolvedValue([message]);
|
||||
mockProducer.send.mockResolvedValue(undefined);
|
||||
|
||||
// Access private poll via casting
|
||||
await (publisher as unknown as { poll: () => Promise<void> }).poll();
|
||||
|
||||
expect(mockProducer.send).toHaveBeenCalledWith({
|
||||
topic: message.topic,
|
||||
messages: [
|
||||
{
|
||||
key: message.aggregateId,
|
||||
value: message.payload,
|
||||
headers: {
|
||||
eventId: message.id,
|
||||
eventType: message.eventType,
|
||||
aggregateType: message.aggregateType,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(outboxRepository.markPublished).toHaveBeenCalledWith(message.id);
|
||||
});
|
||||
|
||||
it("should increment retry when send fails and below max retries", async () => {
|
||||
const message = createMessage({ id: "msg-2", retryCount: 0 });
|
||||
vi.mocked(outboxRepository.findPending).mockResolvedValue([message]);
|
||||
mockProducer.send.mockRejectedValue(new Error("kafka down"));
|
||||
|
||||
await (publisher as unknown as { poll: () => Promise<void> }).poll();
|
||||
|
||||
expect(outboxRepository.incrementRetry).toHaveBeenCalledWith(
|
||||
"msg-2",
|
||||
"kafka down",
|
||||
);
|
||||
expect(outboxRepository.markFailed).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should mark failed when retry count reaches max", async () => {
|
||||
const message = createMessage({ id: "msg-3", retryCount: 4 });
|
||||
vi.mocked(outboxRepository.findPending).mockResolvedValue([message]);
|
||||
mockProducer.send.mockRejectedValue(new Error("kafka down"));
|
||||
|
||||
await (publisher as unknown as { poll: () => Promise<void> }).poll();
|
||||
|
||||
expect(outboxRepository.markFailed).toHaveBeenCalledWith(
|
||||
"msg-3",
|
||||
"kafka down",
|
||||
);
|
||||
expect(outboxRepository.incrementRetry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should handle non-Error rejection in dispatch", async () => {
|
||||
const message = createMessage({ id: "msg-4", retryCount: 0 });
|
||||
vi.mocked(outboxRepository.findPending).mockResolvedValue([message]);
|
||||
mockProducer.send.mockRejectedValue("string error");
|
||||
|
||||
await (publisher as unknown as { poll: () => Promise<void> }).poll();
|
||||
|
||||
expect(outboxRepository.incrementRetry).toHaveBeenCalledWith(
|
||||
"msg-4",
|
||||
"string error",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("poll error handling", () => {
|
||||
it("should swallow errors from findPending", async () => {
|
||||
vi.mocked(outboxRepository.findPending).mockRejectedValue(
|
||||
new Error("db error"),
|
||||
);
|
||||
await expect(
|
||||
(publisher as unknown as { poll: () => Promise<void> }).poll(),
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it("should process multiple messages in a batch", async () => {
|
||||
const messages = [
|
||||
createMessage({ id: "m1" }),
|
||||
createMessage({ id: "m2" }),
|
||||
];
|
||||
vi.mocked(outboxRepository.findPending).mockResolvedValue(messages);
|
||||
mockProducer.send.mockResolvedValue(undefined);
|
||||
|
||||
await (publisher as unknown as { poll: () => Promise<void> }).poll();
|
||||
|
||||
expect(mockProducer.send).toHaveBeenCalledTimes(2);
|
||||
expect(outboxRepository.markPublished).toHaveBeenCalledWith("m1");
|
||||
expect(outboxRepository.markPublished).toHaveBeenCalledWith("m2");
|
||||
});
|
||||
});
|
||||
|
||||
describe("start/stop", () => {
|
||||
it("should start and stop without error", async () => {
|
||||
await publisher.start();
|
||||
await publisher.stop();
|
||||
// Should not throw
|
||||
});
|
||||
|
||||
it("should stop without error when not started", async () => {
|
||||
await publisher.stop();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,81 +0,0 @@
|
||||
import { logger } from "../observability/logger.js";
|
||||
import { producer } from "../../config/kafka.js";
|
||||
import { outboxRepository } from "./outbox.repository.js";
|
||||
import type { OutboxMessage } from "./outbox.schema.js";
|
||||
|
||||
const POLL_INTERVAL_MS = 5000;
|
||||
const BATCH_SIZE = 100;
|
||||
const MAX_RETRY = 5;
|
||||
|
||||
export class OutboxPublisher {
|
||||
private intervalId: ReturnType<typeof setInterval> | null = null;
|
||||
private isPolling = false;
|
||||
|
||||
async start(): Promise<void> {
|
||||
logger.info("OutboxPublisher started");
|
||||
this.intervalId = setInterval(() => {
|
||||
void this.poll();
|
||||
}, POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
}
|
||||
logger.info("OutboxPublisher stopped");
|
||||
}
|
||||
|
||||
private async poll(): Promise<void> {
|
||||
if (this.isPolling) return;
|
||||
this.isPolling = true;
|
||||
try {
|
||||
const messages = await outboxRepository.findPending(BATCH_SIZE);
|
||||
for (const message of messages) {
|
||||
await this.dispatch(message);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error({ error }, "Outbox poll failed");
|
||||
} finally {
|
||||
this.isPolling = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async dispatch(message: OutboxMessage): Promise<void> {
|
||||
try {
|
||||
await producer.send({
|
||||
topic: message.topic,
|
||||
messages: [
|
||||
{
|
||||
key: message.aggregateId,
|
||||
value: message.payload,
|
||||
headers: {
|
||||
eventId: message.id,
|
||||
eventType: message.eventType,
|
||||
aggregateType: message.aggregateType,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await outboxRepository.markPublished(message.id);
|
||||
logger.info(
|
||||
{ id: message.id, eventType: message.eventType, topic: message.topic },
|
||||
"Outbox message published",
|
||||
);
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
logger.error(
|
||||
{ error, id: message.id, eventType: message.eventType },
|
||||
"Outbox publish failed",
|
||||
);
|
||||
if (message.retryCount + 1 >= MAX_RETRY) {
|
||||
await outboxRepository.markFailed(message.id, errorMessage);
|
||||
} else {
|
||||
await outboxRepository.incrementRetry(message.id, errorMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const outboxPublisher = new OutboxPublisher();
|
||||
Reference in New Issue
Block a user