diff --git a/infra/docker-compose.deploy.yml b/infra/docker-compose.deploy.yml index e086e19..c149fbb 100644 --- a/infra/docker-compose.deploy.yml +++ b/infra/docker-compose.deploy.yml @@ -294,11 +294,9 @@ services: environment: PUSH_GATEWAY_PORT: 8081 JWT_SECRET: ${JWT_SECRET} + # M7 (ADR-040): Redis Pub/Sub backplane for real-time push. + # push-gateway no longer mounts Kafka directly. REDIS_URL: ${REDIS_URL} - KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092} - # ARB-013 canonical topic name (v2 alignment with msg Outbox publisher). - KAFKA_NOTIFICATION_TOPIC: edu.notify.notification.sent - KAFKA_CONSUMER_GROUP: push-gateway # PUSH_INTERNAL_TOKEN is the canonical env var (ARB-013 alignment with # msg). INTERNAL_API_TOKEN kept as backward-compat alias. PUSH_INTERNAL_TOKEN: ${PUSH_INTERNAL_TOKEN:-${INTERNAL_API_TOKEN:-edu-internal-token}} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 60b28d9..6f1292f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -182,7 +182,7 @@ importers: version: 5.101.2(react@18.3.0) '@urql/next': specifier: ^1.1.0 - version: 1.1.0(next@14.2.35(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0))(react@18.3.0)(urql@4.2.0(@urql/core@5.0.0(graphql@16.14.2))(react@18.3.0)) + version: 1.1.0(next@14.2.35(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0))(react@18.3.0)(urql@4.2.0(@urql/core@5.0.0(graphql@16.14.2))(react@18.3.0)) clsx: specifier: ^2.1.0 version: 2.1.1 @@ -194,7 +194,7 @@ importers: version: 14.2.35(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0) nuqs: specifier: ^1.19.0 - version: 1.19.0(next@14.2.35(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0)) + version: 1.19.0(next@14.2.35(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0)) react: specifier: ^18.3.0 version: 18.3.0 @@ -343,7 +343,7 @@ importers: version: 3.20.0(next@14.2.35(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0))(react@18.3.0) nuqs: specifier: ^1.19.0 - version: 1.19.0(next@14.2.35(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0)) + version: 1.19.0(next@14.2.35(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0)) react: specifier: ^18.3.0 version: 18.3.0 @@ -832,6 +832,9 @@ importers: graphql: specifier: ^16.9.0 version: 16.14.2 + ioredis: + specifier: ^5.4.0 + version: 5.11.1 kafkajs: specifier: ^2.2.4 version: 2.2.4 @@ -15806,7 +15809,7 @@ snapshots: transitivePeerDependencies: - graphql - '@urql/next@1.1.0(next@14.2.35(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0))(react@18.3.0)(urql@4.2.0(@urql/core@5.0.0(graphql@16.14.2))(react@18.3.0))': + '@urql/next@1.1.0(next@14.2.35(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0))(react@18.3.0)(urql@4.2.0(@urql/core@5.0.0(graphql@16.14.2))(react@18.3.0))': dependencies: next: 14.2.35(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0) react: 18.3.0 @@ -19308,7 +19311,7 @@ snapshots: gauge: 3.0.2 set-blocking: 2.0.0 - nuqs@1.19.0(next@14.2.35(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0)): + nuqs@1.19.0(next@14.2.35(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0)): dependencies: mitt: 3.0.1 next: 14.2.35(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.61.1)(react-dom@18.3.0(react@18.3.0))(react@18.3.0) diff --git a/services/msg/src/channels/in-app.channel.ts b/services/msg/src/channels/in-app.channel.ts index 7c37b14..e478756 100644 --- a/services/msg/src/channels/in-app.channel.ts +++ b/services/msg/src/channels/in-app.channel.ts @@ -8,21 +8,24 @@ import type { * InAppChannel —— 站内信渠道。 * * 站内信数据已在 NotificationService 中写入 MySQL + ES, - * 此渠道仅负责触发 push-gateway 实时推送给在线用户(软失败)。 + * 此渠道仅负责触发实时推送给在线用户(软失败)。 + * + * M7 (ADR-040):实时推送通过 Redis Pub/Sub backplane 发布到 + * `user:{userId}:notify` 通道,realtime-gateway SSE 端点订阅并转发。 * * 仲裁依据 02-architecture-design.md §2.4: * - in_app 总是发送,保证站内信可见 - * - 实时推送通过 push-gateway,不在线时用户下次拉取即可见 + * - 实时推送通过 Redis Pub/Sub,不在线时用户下次拉取即可见 */ -import { sendPush } from "../shared/push/push-gateway.client.js"; +import { publishToRedisPubSub } from "../shared/push/redis-publisher.js"; import { logger } from "../shared/observability/logger.js"; class InAppChannel implements NotificationChannelStrategy { readonly name = "in_app" as const; async send(ctx: ChannelSendContext): Promise { - // 站内信数据已落库,仅触发实时推送 - const pushResult = await sendPush({ + // 站内信数据已落库,仅触发实时推送(M7:Redis Pub/Sub backplane) + const pushResult = await publishToRedisPubSub({ userId: ctx.userId, event: "notification.new", data: { @@ -37,7 +40,7 @@ class InAppChannel implements NotificationChannelStrategy { if (!pushResult.sent) { logger.debug( { userId: ctx.userId, notificationId: ctx.notificationId }, - "In-app push not delivered (user offline or gateway unavailable)", + "In-app real-time push not delivered (user offline or Redis unavailable)", ); } diff --git a/services/msg/src/channels/push.channel.ts b/services/msg/src/channels/push.channel.ts index cd6f476..22cb8f6 100644 --- a/services/msg/src/channels/push.channel.ts +++ b/services/msg/src/channels/push.channel.ts @@ -1,4 +1,4 @@ -import { sendPush } from "../shared/push/push-gateway.client.js"; +import { publishToRedisPubSub } from "../shared/push/redis-publisher.js"; import type { ChannelSendContext, ChannelSendResult, @@ -8,16 +8,16 @@ import type { /** * PushChannel —— 移动推送渠道。 * - * 通过 push-gateway HTTP /internal/push 推送。 - * 仲裁依据 M4:HTTP POST(豁免 gRPC)。 + * M7 (ADR-040):通过 Redis Pub/Sub backplane 发布到 + * `user:{userId}:notify` 通道,realtime-gateway SSE 端点订阅并转发。 * - * 软失败:push-gateway 不可用或用户离线时返回 sent=false,不阻断。 + * 软失败:Redis 不可用或用户离线时返回 sent=false,不阻断。 */ class PushChannel implements NotificationChannelStrategy { readonly name = "push" as const; async send(ctx: ChannelSendContext): Promise { - const result = await sendPush({ + const result = await publishToRedisPubSub({ userId: ctx.userId, event: "notification.push", data: { diff --git a/services/msg/src/shared/push/redis-publisher.ts b/services/msg/src/shared/push/redis-publisher.ts new file mode 100644 index 0000000..517bc82 --- /dev/null +++ b/services/msg/src/shared/push/redis-publisher.ts @@ -0,0 +1,78 @@ +import { getRedis } from "../redis/redis.client.js"; +import { logger } from "../observability/logger.js"; + +/** + * Redis Pub/Sub 实时推送发布器(M7 / ADR-040)。 + * + * 仲裁依据: + * - M7:msg 服务消费 Kafka(持久化、脱敏)后,通过 Redis Pub/Sub 发布实时推送 + * - 通道命名:`user:{userId}:notify` + * - realtime-gateway 实例仅在用户连接 SSE 时 SUBSCRIBE 该通道 + * - 软失败:Redis 不可用时返回 { sent: false },不阻断主流程 + * + * 消息格式:JSON 字符串,由 realtime-gateway SSE 端点原样转发为 `data: {json}\n\n` + */ + +export interface RedisPushRequest { + userId: string; + event: string; + data: Record; +} + +export interface RedisPushResult { + sent: boolean; + error?: string; +} + +/** + * 构造 Redis Pub/Sub 通道名。 + * 通道命名规范(M7):`user:{userId}:notify` + */ +export function notifyChannel(userId: string): string { + return `user:${userId}:notify`; +} + +/** + * 通过 Redis Pub/Sub 发布实时通知到用户的通道。 + * + * 软失败语义: + * - REDIS_URL 未配置 → 返回 { sent: false },不报错 + * - Redis 发布错误 → 返回 { sent: false, error },logger.warn + * - 成功 → 返回 { sent: true } + * + * 消息体为 JSON 字符串,realtime-gateway SSE 端点原样转发为 SSE data 帧。 + */ +export async function publishToRedisPubSub( + req: RedisPushRequest, +): Promise { + const client = getRedis(); + if (!client) { + return { sent: false }; + } + + const channel = notifyChannel(req.userId); + const payload = JSON.stringify({ + event: req.event, + data: req.data, + timestamp: new Date().toISOString(), + }); + + try { + const receivers = await client.publish(channel, payload); + if (receivers === 0) { + // 无订阅者:用户未通过 SSE 连接,不算错误(站内信仍可见) + logger.debug( + { userId: req.userId, channel }, + "Redis Pub/Sub: no active SSE subscribers (user offline)", + ); + } + return { sent: true }; + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + logger.warn( + { err, userId: req.userId, channel }, + "Redis Pub/Sub publish failed", + ); + return { sent: false, error }; + } +} diff --git a/services/push-gateway/go.mod b/services/push-gateway/go.mod index 102edaf..468c5be 100644 --- a/services/push-gateway/go.mod +++ b/services/push-gateway/go.mod @@ -10,7 +10,6 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/prometheus/client_golang v1.23.2 github.com/redis/go-redis/v9 v9.7.0 - github.com/segmentio/kafka-go v0.4.48 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.69.0 go.opentelemetry.io/otel/trace v1.44.0 ) @@ -35,7 +34,6 @@ require ( github.com/goccy/go-yaml v1.19.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.22 // indirect @@ -43,7 +41,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.3.1 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect diff --git a/services/push-gateway/go.sum b/services/push-gateway/go.sum index 9136b80..4faac83 100644 --- a/services/push-gateway/go.sum +++ b/services/push-gateway/go.sum @@ -59,7 +59,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF2 github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -83,8 +82,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -103,8 +100,6 @@ github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs= -github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -120,14 +115,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= -github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8= -github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8= go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -162,53 +149,14 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= diff --git a/services/push-gateway/internal/config/config.go b/services/push-gateway/internal/config/config.go index 366b06b..12a8a35 100644 --- a/services/push-gateway/internal/config/config.go +++ b/services/push-gateway/internal/config/config.go @@ -2,9 +2,11 @@ // variables. Required variables (JWT_SECRET in production, PUSH_INTERNAL_TOKEN // or INTERNAL_API_TOKEN when not in DevMode) cause a fatal exit when missing. // -// v2 alignment (ARB-013 / msg contract): Kafka topic default changed to -// edu.notify.notification.sent; PUSH_INTERNAL_TOKEN is the canonical env var -// for /internal/* auth (INTERNAL_API_TOKEN kept as a backward-compat alias). +// M7 (ADR-040): Kafka config removed — push-gateway no longer mounts Kafka +// directly. Real-time push is routed through Redis Pub/Sub backplane +// (channel: user:{userId}:notify). PUSH_INTERNAL_TOKEN is the canonical env +// var for /internal/* auth (INTERNAL_API_TOKEN kept as a backward-compat +// alias). package config import ( @@ -29,10 +31,6 @@ type Config struct { HeartbeatInterval int // seconds // JWT RS256 (via shared-go/jwks, iam /.well-known/jwks.json) JWKSURL string - // Kafka - KafkaBrokers []string - KafkaNotificationTopic string - KafkaConsumerGroup string // Instance identity (for Redis SET membership) InstanceID string } @@ -80,11 +78,6 @@ func Load() *Config { MaxConnsPerUser: env.GetInt("MAX_CONNS_PER_USER", 5), HeartbeatInterval: env.GetInt("HEARTBEAT_INTERVAL_SECONDS", 30), JWKSURL: env.Get("JWKS_URL", "http://localhost:50052/.well-known/jwks.json"), - KafkaBrokers: parseBrokers(env.Get("KAFKA_BROKERS", "localhost:9092")), - // ARB-013 canonical topic name: edu.... - // msg's Outbox publisher emits to this topic. - KafkaNotificationTopic: env.Get("KAFKA_NOTIFICATION_TOPIC", "edu.notify.notification.sent"), - KafkaConsumerGroup: env.Get("KAFKA_CONSUMER_GROUP", "push-gateway"), InstanceID: env.Get("INSTANCE_ID", generateInstanceID()), } } @@ -105,18 +98,6 @@ func parseOrigins(raw string) []string { return out } -// parseBrokers splits a comma-separated list of Kafka broker addresses. -func parseBrokers(raw string) []string { - parts := strings.Split(raw, ",") - out := make([]string, 0, len(parts)) - for _, p := range parts { - if trimmed := strings.TrimSpace(p); trimmed != "" { - out = append(out, trimmed) - } - } - return out -} - // generateInstanceID returns a stable process-unique identifier. Falls back to // the hostname when INSTANCE_ID is not explicitly set, allowing each replica to // be uniquely identifiable in the Redis online-presence SET. diff --git a/services/push-gateway/internal/config/config_test.go b/services/push-gateway/internal/config/config_test.go index af841b2..b638d69 100644 --- a/services/push-gateway/internal/config/config_test.go +++ b/services/push-gateway/internal/config/config_test.go @@ -32,13 +32,6 @@ func TestLoadDevMode(t *testing.T) { if cfg.HeartbeatInterval != 30 { t.Errorf("HeartbeatInterval = %d, want 30", cfg.HeartbeatInterval) } - // ARB-013: canonical topic name is edu.notify.notification.sent. - if cfg.KafkaNotificationTopic != "edu.notify.notification.sent" { - t.Errorf("KafkaNotificationTopic = %q, want edu.notify.notification.sent", cfg.KafkaNotificationTopic) - } - if cfg.KafkaConsumerGroup != "push-gateway" { - t.Errorf("KafkaConsumerGroup = %q, want push-gateway", cfg.KafkaConsumerGroup) - } if cfg.InstanceID == "" { t.Error("InstanceID should default to hostname, got empty") } @@ -113,17 +106,6 @@ func TestParseOrigins(t *testing.T) { } } -// TestParseBrokers verifies comma-separated broker parsing. -func TestParseBrokers(t *testing.T) { - got := parseBrokers("kafka1:9092,kafka2:9092,kafka3:9092") - if len(got) != 3 { - t.Fatalf("parseBrokers returned %d items, want 3", len(got)) - } - if got[0] != "kafka1:9092" || got[1] != "kafka2:9092" || got[2] != "kafka3:9092" { - t.Errorf("parseBrokers = %v", got) - } -} - // TestGenerateInstanceID verifies fallback to hostname. func TestGenerateInstanceID(t *testing.T) { id := generateInstanceID() diff --git a/services/push-gateway/internal/health/readyz.go b/services/push-gateway/internal/health/readyz.go index 0d640db..5d29888 100644 --- a/services/push-gateway/internal/health/readyz.go +++ b/services/push-gateway/internal/health/readyz.go @@ -6,18 +6,19 @@ // // /readyz reports readiness based on downstream dependency health. Per // ARB-015 §17.4 / ISSUE-058 the readiness probe uses SOFT FAILURE semantics: -// when Redis or Kafka is unavailable the endpoint still returns HTTP 200 but -// carries `degraded: true` plus a `dependencies` block describing which -// component failed. This prevents Kubernetes from evicting the pod when a -// transient dependency blip occurs, at the cost of accepting some degraded -// behavior (no cross-instance fanout, no Kafka consumption) during the blip. -// Only the Hub being in a shutting-down state returns a non-200 (503). +// when Redis is unavailable the endpoint still returns HTTP 200 but carries +// `degraded: true` plus a `dependencies` block describing which component +// failed. This prevents Kubernetes from evicting the pod when a transient +// dependency blip occurs, at the cost of accepting some degraded behavior +// (no Redis Pub/Sub fanout, no SSE delivery) during the blip. Only the Hub +// being in a shutting-down state returns a non-200 (503). +// +// M7 (ADR-040): Kafka probe removed — push-gateway no longer mounts Kafka. // // The hard-failure case is limited to: // - Hub.closing == true (process is shutting down) -> 503 -// - Internal misconfiguration (both Redis and Kafka missing in non-DevMode) -// -> still 200 + degraded, since the pod can still serve local WebSocket -// traffic. +// - Redis missing in non-DevMode -> still 200 + degraded, since the pod can +// still serve local WebSocket traffic. package health import ( @@ -26,7 +27,6 @@ import ( "time" "github.com/edu-cloud/push-gateway/internal/hub" - "github.com/edu-cloud/push-gateway/internal/kafkaconsumer" "github.com/edu-cloud/push-gateway/internal/redisclient" "github.com/gin-gonic/gin" ) @@ -47,12 +47,12 @@ type dependencyStatus struct { // readyzResponse is the ActionState-shaped envelope returned by /readyz. // `degraded` is true when at least one non-critical dependency is unhealthy. type readyzResponse struct { - Status string `json:"status"` - Service string `json:"service"` - InstanceID string `json:"instance_id"` - Degraded bool `json:"degraded"` - Connections int `json:"connections"` - Users int `json:"users"` + Status string `json:"status"` + Service string `json:"service"` + InstanceID string `json:"instance_id"` + Degraded bool `json:"degraded"` + Connections int `json:"connections"` + Users int `json:"users"` Dependencies map[string]*dependencyStatus `json:"dependencies"` } @@ -68,28 +68,26 @@ func Healthz(service string) gin.HandlerFunc { } } -// Readyzer builds the /readyz handler. It probes Redis (PING) and Kafka -// (reader lag / connectivity) and reports degraded state per ARB-015 §17.4. +// Readyzer builds the /readyz handler. It probes Redis (PING) and reports +// degraded state per ARB-015 §17.4. // // The Hub is used to report local connection counts and to detect the // shutting-down state (which triggers a hard 503). The redisClient may be nil -// in DevMode; the kafkaConsumer may be nil when KAFKA_BROKERS is unset. Both -// nil cases are reported as degraded rather than failing the probe. +// in DevMode; the nil case is reported as degraded rather than failing the +// probe. type Readyzer struct { hub *hub.Hub redis *redisclient.Client - kafka *kafkaconsumer.Consumer instance string service string } -// NewReadyzer constructs a Readyzer. redis and kafka may be nil; the resulting -// probe will mark the missing dependency as degraded. -func NewReadyzer(h *hub.Hub, r *redisclient.Client, k *kafkaconsumer.Consumer, service, instanceID string) *Readyzer { +// NewReadyzer constructs a Readyzer. redis may be nil; the resulting probe +// will mark the missing dependency as degraded. +func NewReadyzer(h *hub.Hub, r *redisclient.Client, service, instanceID string) *Readyzer { return &Readyzer{ hub: h, redis: r, - kafka: k, instance: instanceID, service: service, } @@ -98,8 +96,6 @@ func NewReadyzer(h *hub.Hub, r *redisclient.Client, k *kafkaconsumer.Consumer, s // Handler is the gin.HandlerFunc for GET /readyz. func (rz *Readyzer) Handler(c *gin.Context) { // Hard failure: Hub is shutting down — reject new traffic. - // (Hub.CloseAll sets closing=true; we treat this as 503 so the load - // balancer stops sending WebSocket upgrades during drain.) if rz.hub.IsClosing() { c.JSON(http.StatusServiceUnavailable, readyzResponse{ Status: "shutting_down", @@ -115,7 +111,7 @@ func (rz *Readyzer) Handler(c *gin.Context) { return } - deps := make(map[string]*dependencyStatus, 2) + deps := make(map[string]*dependencyStatus, 1) degraded := false // Redis probe (soft failure). @@ -129,17 +125,6 @@ func (rz *Readyzer) Handler(c *gin.Context) { } } - // Kafka probe (soft failure). - if rz.kafka == nil { - deps["kafka"] = &dependencyStatus{Ok: false, Error: "not configured"} - degraded = true - } else { - deps["kafka"] = probeKafka(rz.kafka) - if !deps["kafka"].Ok { - degraded = true - } - } - // Always 200 (unless shutting down) per ISSUE-058/006 soft-failure rule. c.JSON(http.StatusOK, readyzResponse{ Status: statusText(degraded), @@ -171,16 +156,3 @@ func probeRedis(r *redisclient.Client) *dependencyStatus { } return &dependencyStatus{Ok: true, Latency: time.Since(start).Milliseconds()} } - -// probeKafka checks that the consumer reader is still reachable. We use a -// lightweight Lag() call (segmentio/kafka-go Client API); on failure the -// consumer is marked degraded. Note: a degraded Kafka does NOT block WebSocket -// traffic — it only pauses notification consumption until recovery. -func probeKafka(k *kafkaconsumer.Consumer) *dependencyStatus { - ctx, cancel := context.WithTimeout(context.Background(), probeTimeout) - defer cancel() - if err := k.HealthCheck(ctx); err != nil { - return &dependencyStatus{Ok: false, Error: err.Error()} - } - return &dependencyStatus{Ok: true} -} diff --git a/services/push-gateway/internal/kafkaconsumer/consumer.go b/services/push-gateway/internal/kafkaconsumer/consumer.go deleted file mode 100644 index b3efdd2..0000000 --- a/services/push-gateway/internal/kafkaconsumer/consumer.go +++ /dev/null @@ -1,253 +0,0 @@ -// Package kafkaconsumer subscribes to the edu.notify.notification.sent topic -// (ARB-013 canonical topic name) on behalf of push-gateway. Each consumed -// NotificationRequested event is dispatched to the Hub for delivery to online -// clients. At-least-once delivery is enforced via manual commit; idempotency -// is provided by Redis SETNX on event_id (TTL 24h). -// -// Topic naming follows ARB-013 (edu...); the -// previous name edu.notification.requested is deprecated (v2 alignment). -package kafkaconsumer - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "time" - - "github.com/edu-cloud/push-gateway/internal/hub" - "github.com/edu-cloud/push-gateway/internal/observability" - "github.com/edu-cloud/push-gateway/internal/redisclient" - "github.com/segmentio/kafka-go" -) - -// MaxRetries is the per-message retry count before sending to the dead-letter -// topic. After MaxRetries failures the message is committed (skipped) and -// logged; a future P6 enhancement can republish to a DLQ topic. -const MaxRetries = 3 - -// NotificationRequested is the message shape produced by msg (ai10). Field -// names mirror the JSON wire format produced by msg's Outbox publisher. -type NotificationRequested struct { - EventID string `json:"event_id"` - UserID string `json:"user_id"` - EventType string `json:"event_type"` - Channel string `json:"channel"` - Title string `json:"title"` - Content string `json:"content"` - Data json.RawMessage `json:"data"` - Broadcast bool `json:"broadcast"` - OccurredAt int64 `json:"occurred_at"` -} - -// Consumer wraps a kafka.Reader for the notification topic. -type Consumer struct { - reader *kafka.Reader - hub *hub.Hub - redis *redisclient.Client - metrics *observability.Metrics - topic string - groupID string - dlqTopic string -} - -// Config configures the Kafka consumer. -type Config struct { - Brokers []string - Topic string - GroupID string - DLQTopic string -} - -// New creates a Consumer for the configured topic and group. -func New(cfg Config, h *hub.Hub, r *redisclient.Client, m *observability.Metrics) *Consumer { - if cfg.DLQTopic == "" { - cfg.DLQTopic = cfg.Topic + ".dlq" - } - reader := kafka.NewReader(kafka.ReaderConfig{ - Brokers: cfg.Brokers, - Topic: cfg.Topic, - GroupID: cfg.GroupID, - MinBytes: 1, - MaxBytes: 10 * 1024 * 1024, - CommitInterval: 1 * time.Second, // periodic background commit - StartOffset: kafka.LastOffset, // skip historical backlog on first start - }) - return &Consumer{ - reader: reader, - hub: h, - redis: r, - metrics: m, - topic: cfg.Topic, - groupID: cfg.GroupID, - dlqTopic: cfg.DLQTopic, - } -} - -// Run blocks until ctx is canceled, consuming messages and dispatching them. -// Errors are logged but do not stop the consumer unless ctx is canceled. -func (c *Consumer) Run(ctx context.Context) error { - observability.Logger().Info("kafka consumer starting", - "topic", c.topic, "group", c.groupID, "brokers", c.reader.Config().Brokers) - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - msg, err := c.reader.ReadMessage(ctx) - if err != nil { - if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) { - return nil - } - observability.Logger().Warn("kafka read failed", "err", err) - select { - case <-ctx.Done(): - return nil - case <-time.After(2 * time.Second): - } - continue - } - c.metrics.IncKafkaConsumed(c.topic, fmt.Sprintf("%d", msg.Partition)) - c.processMessage(ctx, msg) - } -} - -// Close releases the underlying reader. Should be called after Run returns. -func (c *Consumer) Close() error { - return c.reader.Close() -} - -// HealthCheck verifies that the Kafka broker is reachable. Used by /readyz as -// a soft-failure probe: a failing check marks the consumer as degraded but -// does not return 503 (per ARB-015 §17.4 / ISSUE-058). -// -// The check issues a metadata request for the consumer topic; on success the -// broker is considered reachable. This is cheaper than a Lag() call which -// requires partition assignment to have completed. -func (c *Consumer) HealthCheck(ctx context.Context) error { - conn, err := kafka.DialContext(ctx, "tcp", c.reader.Config().Brokers[0]) - if err != nil { - return fmt.Errorf("kafka: dial broker: %w", err) - } - defer conn.Close() - partitions, err := conn.ReadPartitions(c.topic) - if err != nil { - return fmt.Errorf("kafka: read partitions: %w", err) - } - if len(partitions) == 0 { - return fmt.Errorf("kafka: topic %q has no partitions", c.topic) - } - return nil -} - -// Topic returns the configured topic name. Exposed for diagnostics. -func (c *Consumer) Topic() string { - return c.topic -} - -// GroupID returns the configured consumer group name. -func (c *Consumer) GroupID() string { - return c.groupID -} - -// processMessage handles a single Kafka message with retry and idempotency. -func (c *Consumer) processMessage(ctx context.Context, msg kafka.Message) { - var event NotificationRequested - if err := json.Unmarshal(msg.Value, &event); err != nil { - observability.Logger().Warn("kafka: invalid message payload", - "topic", c.topic, "partition", msg.Partition, "offset", msg.Offset, "err", err) - c.metrics.IncPushed("notification.invalid", "invalid") - // Commit to skip the poison message. - _ = c.reader.CommitMessages(ctx, msg) - return - } - - // eventLabel is the metrics label for IncPushed. Prefer event_type (the - // actual event name like "ExamExtended"); fall back to a generic label. - eventLabel := event.EventType - if eventLabel == "" { - eventLabel = "notification.sent" - } - - // Idempotency: skip already-processed events by event_id. - fresh, err := c.redis.DedupEventId(ctx, event.EventID) - if err != nil { - observability.Logger().Warn("kafka: dedup check failed, processing anyway", - "event_id", event.EventID, "err", err) - } - if !fresh { - observability.Logger().Debug("kafka: duplicate event skipped", - "event_id", event.EventID) - c.metrics.IncPushed(eventLabel, "duplicate") - _ = c.reader.CommitMessages(ctx, msg) - return - } - - // Retry up to MaxRetries on dispatch failure. - for attempt := 1; attempt <= MaxRetries; attempt++ { - if err := c.dispatch(ctx, event); err == nil { - c.metrics.IncPushed(eventLabel, "delivered") - _ = c.reader.CommitMessages(ctx, msg) - return - } else if attempt < MaxRetries { - observability.Logger().Warn("kafka: dispatch retry", - "event_id", event.EventID, "attempt", attempt, "err", err) - time.Sleep(time.Duration(attempt) * time.Second) - } - } - // MaxRetries exhausted: send to DLQ (best-effort) and commit. - observability.Logger().Error("kafka: dispatch failed after retries, sending to DLQ", - "event_id", event.EventID, "topic", c.topic) - c.sendToDLQ(ctx, msg.Value) - c.metrics.IncPushed(eventLabel, "dlq") - _ = c.reader.CommitMessages(ctx, msg) -} - -// dispatch delivers the event to local Hub connections and, when the user is -// not local, asks Redis to fan out via Pub/Sub. -func (c *Consumer) dispatch(ctx context.Context, event NotificationRequested) error { - payload, err := json.Marshal(map[string]any{ - "type": "message", - "event": event.EventType, - "data": event.Data, - "timestamp": time.Now().UTC().Format(time.RFC3339), - }) - if err != nil { - return fmt.Errorf("marshal payload: %w", err) - } - - if event.Broadcast { - // Broadcast: deliver locally + publish to other instances. - c.hub.Broadcast(payload) - return c.redis.PublishBroadcast(ctx, redisclient.CrossInstanceMessage{ - Event: event.EventType, - Data: event.Data, - }) - } - - // Directed: try local first, fall back to Redis Pub/Sub. - if c.hub.HasUser(event.UserID) { - if c.hub.SendToUser(event.UserID, payload) > 0 { - return nil - } - } - return c.redis.PublishUser(ctx, event.UserID, redisclient.CrossInstanceMessage{ - Event: event.EventType, - Data: event.Data, - }) -} - -// sendToDLQ publishes a raw message to the dead-letter topic. Best-effort; -// failure is logged but does not block commit. -func (c *Consumer) sendToDLQ(ctx context.Context, value []byte) { - conn, err := kafka.DialLeader(ctx, "tcp", c.reader.Config().Brokers[0], c.dlqTopic, 0) - if err != nil { - observability.Logger().Error("kafka: DLQ dial failed", "topic", c.dlqTopic, "err", err) - return - } - defer conn.Close() - _ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) - _, _ = conn.WriteMessages(kafka.Message{Value: value}) -} diff --git a/services/push-gateway/internal/redisclient/pubsub.go b/services/push-gateway/internal/redisclient/pubsub.go index 8e5b276..5db2cdf 100644 --- a/services/push-gateway/internal/redisclient/pubsub.go +++ b/services/push-gateway/internal/redisclient/pubsub.go @@ -359,6 +359,16 @@ func idempotentKey(eventID string) string { return "edu:push:idempotent:" + eventID } +// NotifyChannel returns the Redis Pub/Sub channel name for a user's real-time +// notification stream. M7 (ADR-040): msg service publishes notifications to +// this channel; realtime-gateway instances subscribe per-connection when a +// user connects via SSE. +// +// Channel naming: `user:{userId}:notify` +func NotifyChannel(userID string) string { + return "user:" + userID + ":notify" +} + // ParseUserChannel extracts the userID from a "edu:push:channel:user:" // channel name. Returns empty string when the name does not match. func ParseUserChannel(channel string) string { diff --git a/services/push-gateway/internal/sse/handler.go b/services/push-gateway/internal/sse/handler.go new file mode 100644 index 0000000..4556485 --- /dev/null +++ b/services/push-gateway/internal/sse/handler.go @@ -0,0 +1,217 @@ +// Package sse implements the Server-Sent Events endpoint for real-time +// notification delivery. +// +// M7 (ADR-040): The SSE endpoint replaces direct Kafka mounting with a +// Redis Pub/Sub backplane. Each SSE connection dynamically subscribes to +// the Redis channel `user:{userId}:notify` for the authenticated user. +// When the user disconnects, the subscription is released. This per-connection +// subscription model means each realtime-gateway instance only holds Redis +// subscriptions for users actively connected to it, enabling horizontal +// scaling behind a load balancer. +// +// Authentication: +// - JWT RS256 validated via shared-go/jwks (caching JWKS from iam +// /.well-known/jwks.json). DevMode accepts the literal "dev-token" +// returning a synthetic dev-user subject. +// +// SSE format: `data: {json}\n\n` (per the HTML5 Server-Sent Events spec). +// Heartbeat comments (`: heartbeat\n\n`) are sent every 30s to keep the +// connection alive through proxies. +package sse + +import ( + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/edu-cloud/push-gateway/internal/config" + "github.com/edu-cloud/push-gateway/internal/observability" + "github.com/edu-cloud/push-gateway/internal/redisclient" + "github.com/edu-cloud/shared-go/jwks" + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" +) + +// heartbeatInterval is how often a comment frame is sent to keep the SSE +// connection alive through proxies that idle-timeout connections. +const heartbeatInterval = 30 * time.Second + +// errResponse is the ActionState-shaped error envelope returned for auth +// failures and Redis unavailability. +type errResponse struct { + Success bool `json:"success"` + Error *errBody `json:"error,omitempty"` +} + +type errBody struct { + Code string `json:"code"` + Message string `json:"message"` +} + +var ( + errMissingToken = errors.New("missing token") + errInvalidToken = errors.New("invalid token") + errMissingUserID = errors.New("missing user id") +) + +// Handler owns the SSE endpoint for notification delivery. +type Handler struct { + cfg *config.Config + redis *redisclient.Client + jwksFetcher *jwks.Fetcher + metrics *observability.Metrics +} + +// NewHandler constructs an SSE Handler. cfg supplies JWT auth configuration; +// redis may be nil in DevMode (SSE returns 503 when Redis is unavailable). +// jwksFetcher is initialized from cfg.JWKSURL when non-empty. +func NewHandler(cfg *config.Config, r *redisclient.Client, m *observability.Metrics) *Handler { + var fetcher *jwks.Fetcher + if cfg.JWKSURL != "" { + fetcher = jwks.NewFetcher(cfg.JWKSURL) + } + return &Handler{ + cfg: cfg, + redis: r, + jwksFetcher: fetcher, + metrics: m, + } +} + +// HandleNotifications is the GET /sse/notifications endpoint. It upgrades the +// HTTP connection to a persistent SSE stream, subscribing to the user's +// Redis Pub/Sub channel and forwarding messages as SSE data frames. +// +// Lifecycle: +// 1. Authenticate via JWT (query ?token= or Authorization: Bearer header) +// 2. Set SSE headers (Content-Type: text/event-stream, no-cache, etc.) +// 3. SUBSCRIBE to Redis channel `user:{userId}:notify` +// 4. Forward each Redis message as `data: {json}\n\n` +// 5. Send heartbeat comment every 30s +// 6. On client disconnect or context cancel: UNSUBSCRIBE (deferred pubsub.Close) +func (h *Handler) HandleNotifications(c *gin.Context) { + userID, err := h.authenticate(c) + if err != nil { + c.AbortWithStatusJSON(http.StatusUnauthorized, errResponse{ + Success: false, + Error: &errBody{Code: "SSE_UNAUTHORIZED", Message: err.Error()}, + }) + return + } + + if h.redis == nil { + c.AbortWithStatusJSON(http.StatusServiceUnavailable, errResponse{ + Success: false, + Error: &errBody{Code: "SSE_REDIS_UNAVAILABLE", Message: "Redis not configured"}, + }) + return + } + + // SSE response headers. + c.Header("Content-Type", "text/event-stream") + c.Header("Cache-Control", "no-cache") + c.Header("Connection", "keep-alive") + c.Header("X-Accel-Buffering", "no") // Disable nginx buffering. + + // Per-connection Redis subscription to the user's notify channel. + // go-redis PubSub is goroutine-safe per instance; each Subscribe call + // creates an independent PubSub that is closed on connection end. + channel := redisclient.NotifyChannel(userID) + pubsub := h.redis.Redis().Subscribe(c.Request.Context(), channel) + defer pubsub.Close() + + msgCh := pubsub.Channel() + ticker := time.NewTicker(heartbeatInterval) + defer ticker.Stop() + + observability.Logger().Info("sse connection opened", + "user_id", userID, "channel", channel) + + c.Stream(func(w io.Writer) bool { + select { + case <-c.Request.Context().Done(): + return false + case msg, ok := <-msgCh: + if !ok { + // Redis subscription closed (e.g. Redis connection lost). + observability.Logger().Warn("sse redis subscription closed", + "user_id", userID) + return false + } + // Forward Redis payload as SSE data frame: `data: {json}\n\n` + fmt.Fprintf(w, "data: %s\n\n", msg.Payload) + h.metrics.IncPushed("sse.notification", "delivered") + return true + case <-ticker.C: + // SSE comment frame keeps the connection alive through proxies. + fmt.Fprintf(w, ": heartbeat\n\n") + return true + } + }) + + observability.Logger().Info("sse connection closed", "user_id", userID) +} + +// authenticate validates the JWT token. Production uses RS256 via shared-go/jwks; +// DevMode accepts the literal "dev-token" string. The token may be passed via +// the ?token= query parameter or the Authorization: Bearer header. +func (h *Handler) authenticate(c *gin.Context) (string, error) { + tokenStr := c.Query("token") + if tokenStr == "" { + if auth := c.GetHeader("Authorization"); strings.HasPrefix(auth, "Bearer ") { + tokenStr = strings.TrimPrefix(auth, "Bearer ") + } + } + if tokenStr == "" { + return "", errMissingToken + } + + // DevMode shortcut: dev-token maps to a synthetic local user. + if h.cfg.DevMode && tokenStr == "dev-token" { + return "dev-user", nil + } + + // RS256 via shared-go/jwks (preferred, requires iam /.well-known/jwks.json). + if h.jwksFetcher != nil { + claims, err := h.jwksFetcher.ValidateToken(tokenStr) + if err != nil { + return "", errInvalidToken + } + if claims.UserID == "" { + return "", errMissingUserID + } + return claims.UserID, nil + } + + // DevMode without JWKS: fall back to HS256 with the configured secret. + if h.cfg.DevMode && h.cfg.JWTSecret != "" { + return validateHS256(tokenStr, h.cfg.JWTSecret) + } + return "", errInvalidToken +} + +// validateHS256 parses and validates an HS256 JWT signed with secret. Used +// only in DevMode when no JWKS endpoint is configured (local integration +// tests). Returns the user_id claim on success. +func validateHS256(tokenStr, secret string) (string, error) { + claims := &jwks.Claims{} + parsed, err := jwt.ParseWithClaims(tokenStr, claims, func(t *jwt.Token) (any, error) { + if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("sse: unexpected signing method %v", t.Header["alg"]) + } + return []byte(secret), nil + }) + if err != nil { + return "", errInvalidToken + } + if !parsed.Valid { + return "", errInvalidToken + } + if claims.UserID == "" { + return "", errMissingUserID + } + return claims.UserID, nil +} diff --git a/services/push-gateway/main.go b/services/push-gateway/main.go index ed1babd..920646f 100644 --- a/services/push-gateway/main.go +++ b/services/push-gateway/main.go @@ -1,28 +1,41 @@ -// Command push-gateway is the Edu platform's real-time WebSocket push gateway. +// Command push-gateway is the Edu platform's real-time push gateway. // -// It maintains per-user WebSocket connections, delivers directed and broadcast -// messages from msg (via HTTP /internal/* APIs and Kafka), and synchronizes -// online presence across instances via Redis Pub/Sub. +// M7 (ADR-040): The gateway no longer mounts Kafka directly. Real-time push +// is routed through a Redis Pub/Sub backplane. The msg service publishes +// notifications to Redis channel `user:{userId}:notify` after Kafka +// consumption (persistence, desensitization). This gateway dynamically +// subscribes to the user's Redis channel when an SSE connection is opened, +// and unsubscribes on disconnect. +// +// Endpoints: +// - GET /sse/notifications: SSE stream for real-time notifications (JWT auth, +// per-connection Redis subscription). M7 preferred path. +// - GET /ws: WebSocket upgrade (JWT auth, legacy path with global Redis +// PSubscribe fanout for backwards compatibility). +// - POST /internal/push, /internal/broadcast: internal HTTP APIs (X-Internal-Key +// auth, consumed by msg for legacy WebSocket fanout). +// - GET /internal/online/:userID: presence query. +// - GET /healthz, /readyz: health probes. +// - GET /metrics: Prometheus metrics. // // Startup sequence: // 1. config.Load (env vars, DevMode detection) // 2. observability.InitLogger (slog JSON/text) // 3. observability.InitTracer (shared-go/tracer → OTLP) // 4. observability.NewMetrics (Prometheus) -// 5. hub.NewHub (in-memory connection registry) +// 5. hub.NewHub (in-memory connection registry for WebSocket) // 6. redisclient.New + SetHub + RebuildPresenceOnStartup (ISSUE-058) -// 7. redisclient.SubscribeAll (cross-instance Pub/Sub fanout) -// 8. kafkaconsumer.New + goroutine Run (edu.notify.notification.sent, ARB-013) -// 9. ws.NewHandler (JWT RS256, X-Internal-Key, WebSocket upgrade) +// 7. redisclient.SubscribeAll (cross-instance Pub/Sub fanout for WebSocket) +// 8. ws.NewHandler (JWT RS256, X-Internal-Key, WebSocket upgrade) +// 9. sse.NewHandler (JWT RS256, per-connection Redis subscription) // 10. health.NewReadyzer (/readyz soft-failure probe) // 11. gin router + http.Server // // Graceful shutdown (SIGINT/SIGTERM): -// 1. Hub.CloseAll (send close frame 1001 to every live connection) -// 2. http.Server.Shutdown (stop accepting new HTTP/WebSocket requests) -// 3. Kafka consumer goroutine cancel + reader.Close -// 4. Redis Pub/Sub cancel + client.Close -// 5. Tracer shutdown (flush pending spans) +// 1. Hub.CloseAll (send close frame 1001 to every live WebSocket connection) +// 2. http.Server.Shutdown (stop accepting new HTTP/SSE/WebSocket requests) +// 3. Redis Pub/Sub cancel + client.Close +// 4. Tracer shutdown (flush pending spans) package main import ( @@ -37,9 +50,9 @@ import ( "github.com/edu-cloud/push-gateway/internal/config" "github.com/edu-cloud/push-gateway/internal/health" "github.com/edu-cloud/push-gateway/internal/hub" - "github.com/edu-cloud/push-gateway/internal/kafkaconsumer" "github.com/edu-cloud/push-gateway/internal/observability" "github.com/edu-cloud/push-gateway/internal/redisclient" + "github.com/edu-cloud/push-gateway/internal/sse" "github.com/edu-cloud/push-gateway/internal/ws" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -50,7 +63,8 @@ const serviceName = "push-gateway" // shutdownTimeout bounds the graceful shutdown of the HTTP server. After this // window in-flight requests are forcibly closed. WebSocket long-poll -// connections are drained by Hub.CloseAll before this timer starts. +// connections and SSE streams are drained by Hub.CloseAll before this timer +// starts; SSE streams are drained by http.Server.Shutdown. const shutdownTimeout = 10 * time.Second func main() { @@ -65,8 +79,6 @@ func main() { "max_conns_per_user", cfg.MaxConnsPerUser, "jwks_url", cfg.JWKSURL, "redis_url", cfg.RedisURL, - "kafka_brokers", cfg.KafkaBrokers, - "kafka_topic", cfg.KafkaNotificationTopic, ) // 2. OpenTelemetry tracer (best-effort). @@ -76,7 +88,7 @@ func main() { // 3. Prometheus metrics. metrics := observability.NewMetrics() - // 4. Hub: in-memory connection registry. + // 4. Hub: in-memory connection registry (for WebSocket legacy path). h := hub.NewHub(cfg.MaxConnsPerUser) // 5. Redis client (optional in DevMode). When present, wire presence @@ -98,10 +110,10 @@ func main() { logger.Info("redis presence synced", "url", cfg.RedisURL) } } else { - logger.Warn("REDIS_URL empty; cross-instance fanout disabled") + logger.Warn("REDIS_URL empty; SSE and cross-instance fanout disabled") } - // 6. Redis Pub/Sub subscriber (cross-instance message fanout). + // 6. Redis Pub/Sub subscriber (cross-instance message fanout for WebSocket). var pubsubCancel func() error if redisClient != nil { subCtx, subCancel := context.WithCancel(context.Background()) @@ -116,36 +128,14 @@ func main() { } } - // 7. Kafka consumer (edu.notify.notification.sent, ARB-013). Started only - // when at least one broker is configured; otherwise the HTTP /internal/push - // API is the only delivery channel. - var kafkaConsumer *kafkaconsumer.Consumer - var kafkaCancel context.CancelFunc - if len(cfg.KafkaBrokers) > 0 && cfg.KafkaNotificationTopic != "" { - kafkaConsumer = kafkaconsumer.New(kafkaconsumer.Config{ - Brokers: cfg.KafkaBrokers, - Topic: cfg.KafkaNotificationTopic, - GroupID: cfg.KafkaConsumerGroup, - }, h, redisClient, metrics) - kafkaCtx, cancel := context.WithCancel(context.Background()) - kafkaCancel = cancel - go func() { - logger.Info("kafka consumer starting", - "topic", cfg.KafkaNotificationTopic, "group", cfg.KafkaConsumerGroup) - if err := kafkaConsumer.Run(kafkaCtx); err != nil && - !errors.Is(err, context.Canceled) { - logger.Error("kafka consumer exited with error", "err", err) - } - }() - } else { - logger.Warn("KAFKA_BROKERS empty; notification consumption disabled") - } - - // 8. WebSocket + internal HTTP handlers. + // 7. WebSocket + internal HTTP handlers (legacy path, backwards compat). wsHandler := ws.NewHandler(h, cfg, redisClient, metrics) + // 8. SSE handler (M7 preferred path: per-connection Redis subscription). + sseHandler := sse.NewHandler(cfg, redisClient, metrics) + // 9. /readyz probe (soft failure per ARB-015 §17.4). - readyzer := health.NewReadyzer(h, redisClient, kafkaConsumer, serviceName, cfg.InstanceID) + readyzer := health.NewReadyzer(h, redisClient, serviceName, cfg.InstanceID) // 10. Gin router. gin.SetMode(gin.ReleaseMode) @@ -155,10 +145,12 @@ func main() { // Liveness (no auth, no dependency checks). r.GET("/healthz", health.Healthz(serviceName)) - // Readiness (soft failure on Redis/Kafka). + // Readiness (soft failure on Redis). r.GET("/readyz", readyzer.Handler) // Prometheus metrics. r.GET("/metrics", gin.WrapH(promhttp.Handler())) + // SSE endpoint (M7 preferred: per-connection Redis subscription). + r.GET("/sse/notifications", sseHandler.HandleNotifications) // WebSocket upgrade (JWT auth via query ?token= or Authorization header). r.GET("/ws", wsHandler.HandleWebSocket) // Internal HTTP APIs consumed by msg (X-Internal-Key auth, ARB-013). @@ -171,7 +163,7 @@ func main() { Addr: ":" + cfg.Port, Handler: r, ReadTimeout: 10 * time.Second, - WriteTimeout: 0, // WebSocket connections are long-lived; no write timeout. + WriteTimeout: 0, // SSE and WebSocket connections are long-lived; no write timeout. } // 11. Start HTTP server. @@ -193,27 +185,17 @@ func main() { shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout) defer shutdownCancel() - // 13a. Hub.CloseAll sends close frame 1001 to every live connection. + // 13a. Hub.CloseAll sends close frame 1001 to every live WebSocket connection. h.CloseAll() logger.Info("hub closeAll complete; draining connections", "active_connections", h.ActiveConnections()) - // 13b. Stop accepting new HTTP requests; drain in-flight. + // 13b. Stop accepting new HTTP requests; drain in-flight (including SSE). if err := srv.Shutdown(shutdownCtx); err != nil { logger.Error("http server shutdown error", "err", err) } - // 13c. Stop Kafka consumer (cancel context + close reader). - if kafkaCancel != nil { - kafkaCancel() - } - if kafkaConsumer != nil { - if err := kafkaConsumer.Close(); err != nil { - logger.Warn("kafka consumer close error", "err", err) - } - } - - // 13d. Stop Redis Pub/Sub subscriber + close client. + // 13c. Stop Redis Pub/Sub subscriber + close client. if pubsubCancel != nil { if err := pubsubCancel(); err != nil { logger.Warn("redis pubsub cancel error", "err", err)