From 3fac472a577daed8f7c31e08bf73fbac06a99611 Mon Sep 17 00:00:00 2001
From: SpecialX <47072643+wangxiner55@users.noreply.github.com>
Date: Fri, 10 Jul 2026 17:36:53 +0800
Subject: [PATCH] chore(push-gateway): ai02 module updates - config, hub, ws,
kafka, health, docs
---
docs/modules/push-gateway/README.md | 186 ++++--
docs/troubleshooting/known-issues.md | 31 +-
go.work.sum | 2 +
services/push-gateway/Dockerfile | 50 +-
services/push-gateway/README.md | 98 +++-
.../docs/02-architecture-design.md | 306 ++++++++--
services/push-gateway/go.mod | 11 +-
services/push-gateway/go.sum | 60 ++
.../internal/config/config_test.go | 120 ++++
.../push-gateway/internal/health/readyz.go | 186 ++++++
services/push-gateway/internal/hub/hub.go | 25 +-
.../push-gateway/internal/hub/hub_test.go | 238 ++++++++
.../internal/kafkaconsumer/consumer.go | 33 ++
services/push-gateway/internal/ws/handler.go | 540 ++++++++++++++----
.../push-gateway/internal/ws/handler_test.go | 196 +++++++
services/push-gateway/main.go | 224 ++++++--
16 files changed, 2017 insertions(+), 289 deletions(-)
create mode 100644 services/push-gateway/internal/config/config_test.go
create mode 100644 services/push-gateway/internal/health/readyz.go
create mode 100644 services/push-gateway/internal/hub/hub_test.go
create mode 100644 services/push-gateway/internal/ws/handler_test.go
diff --git a/docs/modules/push-gateway/README.md b/docs/modules/push-gateway/README.md
index 5809c9f..093927f 100644
--- a/docs/modules/push-gateway/README.md
+++ b/docs/modules/push-gateway/README.md
@@ -1,32 +1,42 @@
# push-gateway 推送网关服务
-> 版本：0.1（骨架）
-> 日期：2026-07-07
-> 状态：P5 待交付
-> 关联文档：[架构影响地图](../../architecture/004_architecture_impact_map.md)
+> 版本：1.0（仲裁回写）
+> 日期：2026-07-10
+> 状态：P5 已交付
+> 关联文档：
+>
+> - [架构影响地图](../../architecture/004_architecture_impact_map.md)
+> - [02 架构设计](../../../services/push-gateway/docs/02-architecture-design.md)
+> - [president-final-rulings](../../architecture/president-final-rulings.md)
---
## 1. 模块职责
-push-gateway 是实时推送基础设施服务（Go 实现），管理 WebSocket/SSE 长连接。
-接收 msg 服务的推送请求，维护用户在线连接池，将消息实时投递到浏览器/移动端。
-支持连接鉴权、心跳保活、断线重连、多设备同步、连接数监控。
+push-gateway 是实时推送基础设施服务（Go 实现），管理 WebSocket 长连接（**仅 WebSocket，不支持 SSE** — ISSUE-001）。
+接收 msg 服务的推送请求（HTTP `/internal/*` + Kafka 事件），维护用户在线连接池，将消息实时投递到浏览器/移动端。
+支持连接鉴权（JWT RS256）、RFC 6455 控制帧心跳、断线重连、多设备同步、连接数监控、跨实例 Pub/Sub 广播。
+
+**无 DB**（ISSUE-005）：消息审计、离线落库、重试队列均由 msg 服务负责。
---
## 2. 技术栈
-| 类别 | 技术 | 版本 | 用途 |
-|------|------|------|------|
-| 语言 | Go | 1.22+ | 高并发长连接 |
-| Web 框架 | Gin | 1.10+ | HTTP 升级 WebSocket |
-| WebSocket | gorilla/websocket | 1.5+ | WebSocket 协议 |
-| 缓存 | Redis | 7.x | 在线连接表、pub/sub |
-| 配置 | viper | 1.18+ | 配置注入 |
-| 日志 | zap | 1.27+ | 结构化日志 |
-| 追踪 | OpenTelemetry | 1.24+ | 分布式追踪 |
-| 编排 | Kubernetes | 1.28+ | 生产部署 |
+| 类别 | 技术 | 版本 | 用途 |
+| --------- | ------------------------ | ------ | -------------------------------------------------- |
+| 语言 | Go | 1.25+ | 高并发长连接 |
+| Web 框架 | Gin | 1.12+ | HTTP 升级 WebSocket + 内部 API |
+| WebSocket | gorilla/websocket | 1.5+ | WebSocket 协议 |
+| 缓存 | Redis | 7.x | 在线连接表、pub/sub 跨实例 fanout |
+| Kafka | segmentio/kafka-go | 0.4+ | 消费 `edu.notification.requested`（纯 Go，无 cgo） |
+| JWT | golang-jwt/jwt/v5 | 5.2+ | RS256 token 校验（via shared-go/jwks） |
+| 配置 | shared-go/env | — | 环境变量加载（替代 viper） |
+| 日志 | log/slog | stdlib | 结构化日志（JSON prod / text dev） |
+| 追踪 | OpenTelemetry | 1.44+ | 分布式追踪（via shared-go/tracer） |
+| 指标 | prometheus/client_golang | 1.23+ | Prometheus 指标 |
+| 共享包 | shared-go | — | tracer/logger/env/jwks 复用（ARB-015 §17.5） |
+| 编排 | Kubernetes | 1.28+ | 生产部署 |
---
@@ -34,17 +44,20 @@ push-gateway 是实时推送基础设施服务（Go 实现），管理 WebSocket
```mermaid
graph TB
- MSG[msg 服务] -->|gRPC Push| PG[push-gateway]
- PG --> HUB[ConnectionHub
连接池管理]
+ MSG[msg 服务] -->|HTTP /internal/push| PG[push-gateway]
+ MSG -->|Kafka edu.notification.requested| PG
+ PG --> HUB[Hub
连接池管理
map userID -> connID -> Connection]
HUB --> C1[用户 1 连接]
HUB --> C2[用户 2 连接]
HUB --> CN[用户 N 连接]
- PG -->[(Redis
在线连接表 + pub/sub)]
- MFE[微前端] -.WebSocket.-> PG
- PG --> HEART[心跳保活
30s ping/pong]
+ PG --> REDIS[(Redis
在线 SET + pub/sub)]
+ MFE[微前端] -.WebSocket /ws?token=JWT.-> PG
+ PG --> HEART[心跳保活
RFC 6455 Ping/Pong 30s]
+ PG --> JWKS[iam JWKS
RS256 公钥 5min 缓存]
+ PG --> METRICS[/metrics
9 个 Prometheus 指标]
```
-> 待 P5 交付时补充完整的分层架构图与多副本连接同步图。
+> 完整分层图、跨实例同步图、失败模式矩阵见 [02 架构设计](../../../services/push-gateway/docs/02-architecture-design.md) §1 / §8 / §16。
---
@@ -57,58 +70,125 @@ sequenceDiagram
participant R as Redis
participant MSG as msg 服务
- U->>PG: WebSocket 连接 + JWT 鉴权
- PG->>R: 注册在线连接（userId → nodeId）
+ U->>PG: WebSocket /ws?token=JWT
+ PG->>PG: JWT RS256 校验（via JWKS）
+ PG->>HUB: Register(userID, conn)
+ HUB->>R: SADD edu:push:online: + EXPIRE 60s
PG-->>U: 连接建立
- Note over PG: 心跳保活 30s ping/pong
+ Note over PG: RFC 6455 Ping/Pong 30s；60s 无帧断开
- MSG->>PG: gRPC Push(userId, payload)
- PG->>R: 查询 userId 在线 nodeId
- alt 本节点在线
- PG->>U: WebSocket 推送消息
- else 跨节点在线
- PG->>R: pub/sub 转发到目标节点
- R->>PG: 目标节点订阅收到
- PG->>U: 推送消息
+ MSG->>PG: POST /internal/push {user_id, event, data}
+ PG->>HUB: SendToUser(userID, msg)
+ alt 本实例有此用户
+ PG->>U: WebSocket 推送
+ PG-->>MSG: {delivered:true, online:true}
+ else 本实例无此用户
+ PG->>R: SMEMBERS edu:push:online:
+ alt 用户在线（其他实例）
+ PG->>R: PUBLISH edu:push:channel:user.
+ R-->>PG: 持有实例订阅后投递
+ PG-->>MSG: {delivered:true, online:true}
+ else 用户离线
+ PG-->>MSG: {delivered:false, online:false}
+ Note over MSG: msg 走离线推送（SMS/邮件）
+ end
end
```
-> 待 P5 交付时补充：断线重连流程、多设备同步流程、连接数监控流程。
-
---
## 5. 对外契约
-- **gRPC 服务**：PushService（Push、Broadcast、GetOnlineStatus）（待 P5 定义 protobuf）
-- **HTTP 端点**：`/ws`（WebSocket 升级）、`/healthz`、`/readyz`、`/metrics`
-- **事件**：不发布也不消费 Kafka 事件，仅接收 msg 的 gRPC 推送调用
-- **Redis 协议**：在线连接表（HASH）、跨节点推送（pub/sub channel `push:{nodeId}`）
+### 5.1 HTTP 端点
-> 待 P5 交付时补充完整 protobuf 定义与 WebSocket 消息协议。
+| 方法 | 路径 | 鉴权 | 用途 |
+| ---- | --------------------------- | ------------------------------------------------------- | ----------------------------------- |
+| GET | `/ws` | JWT RS256（query `?token=` 或 `Authorization: Bearer`） | WebSocket 升级 |
+| POST | `/internal/push` | `X-Internal-Token` 头（ISSUE-002） | 定向推送 |
+| POST | `/internal/broadcast` | `X-Internal-Token` 头 | 广播 |
+| GET | `/internal/online/` | `X-Internal-Token` 头 | 查在线状态 |
+| GET | `/healthz` | 无 | liveness 探针 |
+| GET | `/readyz` | 无 | readiness 探针（软失败，ISSUE-058） |
+| GET | `/metrics` | 无 | Prometheus 指标 |
+
+### 5.2 Kafka 消费
+
+| Topic | Consumer Group | Message | 说明 |
+| ---------------------------- | -------------- | ----------------------- | ------------------------------------------------ |
+| `edu.notification.requested` | `push-gateway` | `NotificationRequested` | ISSUE-053 重命名（原 `edu.notification.events`） |
+
+- 至少一次 + Redis SETNX 幂等去重（event_id，TTL 24h）
+- 失败重试 3 次后入 DLQ（`edu.notification.requested.dlq`）
+
+### 5.3 Redis 数据结构
+
+| Key 模式 | 类型 | TTL | 用途 |
+| -------------------------------- | ----------------- | --- | -------------------- |
+| `edu:push:online:` | SET（instanceID） | 60s | 在线用户所在实例集合 |
+| `edu:push:channel:user:` | Pub/Sub | — | 跨实例定向推送 |
+| `edu:push:channel:broadcast` | Pub/Sub | — | 跨实例广播 |
+| `edu:push:idempotent:` | SETNX | 24h | Kafka 幂等去重 |
+
+> **无 gRPC 服务**（ISSUE-007）：push-gateway 仅暴露 HTTP + WebSocket，不实现 gRPC 服务，`packages/shared-proto/proto/` 无 push-gateway.proto。
---
## 6. 依赖关系
-- **上游**：msg 服务（gRPC Push 调用）
-- **下游**：Redis（在线连接表、pub/sub 跨节点同步）
-- **客户端**：teacher-portal、student-portal、parent-portal（WebSocket 连接）
-- **共享包**：shared-proto、shared-go
+- **上游**：msg 服务（HTTP `/internal/*` + Kafka 事件）
+- **下游**：Redis（在线 SET + pub/sub 跨实例同步）、iam（JWKS 公钥）
+- **客户端**：teacher-portal、student-portal、parent-portal、admin-portal（WebSocket 连接）
+- **共享包**：shared-go（tracer/logger/env/jwks）
---
## 7. 架构约束
1. 无业务逻辑，仅做连接管理与消息转发
-2. 多副本部署下，通过 Redis pub/sub 同步跨节点推送
-3. 连接鉴权必须校验 JWT，禁止未认证连接
-4. 心跳超时 60s 无响应则断开连接
-5. 单节点最大连接数 50k，超出时拒绝新连接
-
-> 待 P5 交付时补充完整约束清单。
+2. **无 DB**（ISSUE-005）：审计/落库/重试由 msg 负责
+3. **仅 WebSocket**（ISSUE-001）：不支持 SSE
+4. **无 gRPC**（ISSUE-007）：仅 HTTP + WebSocket
+5. 连接鉴权必须校验 JWT RS256（DevMode 接受 `dev-token`）
+6. RFC 6455 控制帧心跳（Ping/Pong 30s，60s 无帧断开）
+7. 单用户连接数上限 5（默认，可配 `MAX_CONNS_PER_USER`）
+8. **单实例容量 10w+**（ISSUE-003，原 50k 已提升）
+9. Redis 故障软失败（ISSUE-058/006）：`/readyz` 返 200 + `degraded:true`，不断本地 WebSocket
+10. Redis online SET 启动重建（ISSUE-058）：先 SREM 旧 instanceID，再基于内存重建
---
-## 8. 架构决策
+## 8. 关键 ADR（完整见 02 §14）
-> 待 P5 交付时补充 ADR 记录，预计包括：Go+gorilla/websocket 选型、Redis pub/sub 跨节点方案、连接数上限策略、心跳间隔选型。
+| ADR | 决策 | 来源 |
+| ------- | ---------------------------------------- | ---------------- |
+| ADR-001 | 仅 WebSocket，不支持 SSE | ISSUE-001 |
+| ADR-002 | `X-Internal-Token` 命名 | ISSUE-002 |
+| ADR-003 | 单实例 10w+ 连接 | ISSUE-003 |
+| ADR-004 | 无 DB，审计由 msg 负责 | ISSUE-005 |
+| ADR-005 | Kafka topic `edu.notification.requested` | ISSUE-053 |
+| ADR-006 | `/readyz` 软失败 | ISSUE-058/006 |
+| ADR-007 | Redis SET 启动重建 | ISSUE-058 |
+| ADR-008 | Redis 必需但软失败 | ISSUE-055 vs 058 |
+| ADR-009 | 纯 HTTP + WebSocket，无 gRPC | ISSUE-007 |
+| ADR-010 | shared-go 复用 | ARB-015 §17.5 |
+
+---
+
+## 9. 环境变量
+
+| 变量 | 默认值 | 说明 |
+| ----------------------------- | ---------------------------------------------- | ---------------------------------------- |
+| `PUSH_GATEWAY_PORT` | 8081 | HTTP 监听端口 |
+| `DEV_MODE` | false | 开发模式（跳过内部鉴权，接受 dev-token） |
+| `JWT_SECRET` | （DevMode 有默认） | JWT 密钥（生产必填） |
+| `JWKS_URL` | `http://localhost:50052/.well-known/jwks.json` | iam JWKS 端点 |
+| `INTERNAL_API_TOKEN` | （生产必填） | `/internal/*` 鉴权 token（ISSUE-002） |
+| `REDIS_URL` | `redis://localhost:6379/0` | Redis 连接 URL |
+| `KAFKA_BROKERS` | `localhost:9092` | Kafka broker 列表（逗号分隔） |
+| `KAFKA_NOTIFICATION_TOPIC` | `edu.notification.requested` | 消费 topic（ISSUE-053） |
+| `KAFKA_CONSUMER_GROUP` | `push-gateway` | consumer group |
+| `WS_ALLOWED_ORIGINS` | （空） | WebSocket Origin 白名单（逗号分隔） |
+| `MAX_CONNS_PER_USER` | 5 | 单用户最大连接数 |
+| `HEARTBEAT_INTERVAL_SECONDS` | 30 | 心跳间隔 |
+| `INSTANCE_ID` | （hostname） | 实例标识（Redis SET 成员） |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | `localhost:4318` | OTLP 端点 |
diff --git a/docs/troubleshooting/known-issues.md b/docs/troubleshooting/known-issues.md
index e3b072e..716fe87 100644
--- a/docs/troubleshooting/known-issues.md
+++ b/docs/troubleshooting/known-issues.md
@@ -364,14 +364,29 @@
### 2.8 push-gateway（Go，P5）
-| 场景 | 技术/规则 |
-| ---------------- | ---------------------------------------------------------------- |
-| WebSocket 长连接 | 单节点支撑 10w+ 连接，业务服务只需调 /internal/push |
-| 跨实例同步 | Redis PubSub（RedisURL 配置，预留 P6 实现） |
-| 离线消息 | 仅推在线用户，离线消息存 MySQL，上线时拉取 |
-| 并发写修复 | send chan + 单写协程模式，避免 gorilla/websocket 并发写竞争 |
-| DEV_MODE 鉴权 | DEV_MODE=true 时接受 dev-token，生产环境必须 JWT 校验 |
-| 广播端点 | POST /internal/broadcast，body {event, data}，调用 hub.Broadcast |
+| 场景 | 技术/规则 |
+| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
+| WebSocket 长连接 | 单实例支撑 10w+ 连接（ISSUE-003），业务服务调 /internal/push 或消费 Kafka |
+| 心跳协议 | RFC 6455 控制帧（Ping/Pong），30s 间隔，SetReadDeadline(60s) 超时断开；禁止文本消息 ping/pong |
+| 单用户连接数限制 | 默认 5，超限返 PUSH_TOO_MANY_CONNECTIONS + close frame 1008 |
+| 跨实例同步 | Redis Pub/Sub 订阅 `edu:push:channel:user:` + `edu:push:channel:broadcast`；在线 SET `edu:push:online:` 60s TTL |
+| Redis SET 启动重建（ISSUE-058） | 启动时 SREM 旧 instanceID 成员，基于内存连接 ForEachUser 重新 SADD，避免 Pod 重启后残留幽灵在线状态 |
+| 离线消息 | 仅推在线用户；离线消息落库由 msg 服务负责（ISSUE-005），push-gateway 无 DB |
+| 并发写修复 | send chan + 单写协程模式，避免 gorilla/websocket 并发写竞争 |
+| 内部 API 鉴权 | `X-Internal-Token` 头 + `INTERNAL_API_TOKEN` 环境变量（ISSUE-002，原名 X-Internal-Key 已废弃） |
+| Kafka topic 命名（ISSUE-053） | `edu.notification.requested`（非 edu.notification.events，遵循 G16 `..` 规则） |
+| Kafka 幂等性 | 基于 event_id Redis SETNX 去重（24h TTL），at-least-once + 消费失败 3 次入 DLQ |
+| /readyz 软失败（ISSUE-058/006） | Redis/Kafka 不可达返 200 + `degraded: true`，仅 Hub.CloseAll 触发 closing 时返 503；避免 K8s 在依赖抖动时驱逐 Pod |
+| 无 gRPC（ISSUE-007） | 仅 HTTP + WebSocket，不暴露 gRPC 服务 |
+| shared-go 复用（ARB-015 §17.5） | tracer/logger/jwks/env 走 shared-go 包，不重复实现 |
+| 错误码前缀 | `PUSH_*`（PUSH_UNAUTHORIZED/PUSH_INVALID_REQUEST/PUSH_TOO_MANY_CONNECTIONS 等） |
+| 多阶段 Dockerfile | golang:1.25-alpine 构建 → alpine:3.20 运行，CGO_ENABLED=0 + ldflags -s -w + non-root user + wget healthcheck |
+| slog 结构化日志 | log/slog（JSON prod / text dev），替代标准 log 包；字段含 request_id/trace_id/user_id/conn_id/event |
+| DEV_MODE 鉴权 | DEV_MODE=true 时接受 dev-token（HS256 fallback），生产环境必须 JWT RS256（JWKS 5min 缓存） |
+| 广播端点 | POST /internal/broadcast，body {event, data}，调用 hub.Broadcast |
+| gorilla/websocket API 陷阱 | SetReadLimit 返回 void（不可 `_ =` 赋值）；SetPongHandler 签名为 `func(appData string) error`（非 `func(string, error) error`） |
+| sync.RWMutex 重入死锁 | 持写锁（Lock）的 goroutine 不可再调用同结构体的 RLock 方法（Go RWMutex 非重入）；CloseAll/Broadcast 内部计数改为内联遍历 |
+| httptest.Server.Close() 死锁 | 测试 handler 阻塞 ReadMessage 时 Close() 会死锁；用 done channel 模式：cleanup 先 close(done) → cli.Close() → srv.Close() |
### 2.9 ai-gateway（Python/FastAPI，P5）
diff --git a/go.work.sum b/go.work.sum
index 6123523..791ba71 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -19,9 +19,11 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs=
github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
diff --git a/services/push-gateway/Dockerfile b/services/push-gateway/Dockerfile
index 79b7d59..eb77580 100644
--- a/services/push-gateway/Dockerfile
+++ b/services/push-gateway/Dockerfile
@@ -1,13 +1,43 @@
-FROM golang:1.22-alpine AS builder
-WORKDIR /app
-COPY go.mod go.sum ./
-RUN go mod download
-COPY . .
-RUN CGO_ENABLED=0 go build -o push-gateway .
+# 多阶段构建：push-gateway 生产镜像
+# 用法：docker build -t edu/push-gateway:latest -f services/push-gateway/Dockerfile .
-FROM alpine:3.20
-RUN apk --no-cache add ca-certificates
+# ============ Builder ============
+FROM golang:1.25-alpine AS builder
WORKDIR /app
-COPY --from=builder /app/push-gateway .
+
+# git 与 ca-certificates 为 go mod 下载所需
+RUN apk add --no-cache git ca-certificates
+
+# 利用 go.work 时需拷贝根 go.work 与 shared-go 包
+COPY go.mod go.sum ./
+COPY packages/shared-go ./packages/shared-go
+COPY services/push-gateway/go.mod services/push-gateway/go.sum ./services/push-gateway/
+RUN cd services/push-gateway && go mod download
+
+# 拷源码并构建（静态编译便于 alpine 运行）
+COPY services/push-gateway ./services/push-gateway
+COPY packages/shared-go ./packages/shared-go
+RUN CGO_ENABLED=0 GOOS=linux go build \
+ -ldflags="-s -w -X main.serviceName=push-gateway" \
+ -o /app/bin/push-gateway \
+ ./services/push-gateway
+
+# ============ Runtime ============
+FROM alpine:3.20 AS runner
+WORKDIR /app
+
+RUN apk add --no-cache ca-certificates tzdata wget
+
+# 非 root 用户
+RUN addgroup -g 1001 -S app && adduser -S app -u 1001 -G app
+
+COPY --from=builder /app/bin/push-gateway /app/push-gateway
+
+USER app
EXPOSE 8081
-CMD ["./push-gateway"]
+
+# 健康检查（liveness probe；readiness 由 /readyz 单独探活）
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+ CMD wget --quiet --spider http://localhost:8081/healthz || exit 1
+
+ENTRYPOINT ["/app/push-gateway"]
diff --git a/services/push-gateway/README.md b/services/push-gateway/README.md
index 4f03547..5a51db7 100644
--- a/services/push-gateway/README.md
+++ b/services/push-gateway/README.md
@@ -1,42 +1,104 @@
# Push Gateway 推送网关服务
-> 版本：0.1（P5 骨架）
+> 版本：1.0（P5 仲裁回写）
> 端口：8081
+> 关联：[02 架构设计](./docs/02-architecture-design.md)、[modules/push-gateway/README](../../docs/modules/push-gateway/README.md)
## 职责
-实时推送基础设施服务（Go 实现），管理 WebSocket 长连接。
-接收 Msg 服务的推送请求，维护用户在线连接池，将消息实时投递到浏览器/移动端。
+实时推送基础设施服务（Go 实现），管理 WebSocket 长连接（仅 WebSocket，不支持 SSE — ISSUE-001）。
+接收 msg 服务的推送请求（HTTP `/internal/*` + Kafka `edu.notification.requested`），维护用户在线连接池，
+将消息实时投递到浏览器/移动端。无 DB（ISSUE-005），审计/落库由 msg 负责。
## 技术栈
-- Go 1.22 + Gin 1.10
-- gorilla/websocket 1.5
-- golang-jwt/jwt/v5（JWT 鉴权）
-- zap（结构化日志，骨架）
+- Go 1.25+ / Gin 1.12+
+- gorilla/websocket 1.5（RFC 6455 控制帧心跳）
+- segmentio/kafka-go 0.4（纯 Go，无 cgo）
+- redis/go-redis/v9（在线 SET + Pub/Sub 跨实例 fanout）
+- golang-jwt/jwt/v5 + shared-go/jwks（RS256 校验，5min 缓存）
+- log/slog（结构化日志，JSON prod / text dev）
+- prometheus/client_golang（9 个指标）
+- shared-go（tracer/logger/env/jwks 复用 — ARB-015 §17.5）
## 开发
```bash
go mod tidy
go run main.go # :8081
+
+# DevMode（跳过内部鉴权，接受 dev-token）
+DEV_MODE=true go run main.go
+
+# 生产模式（需设置 JWT_SECRET + INTERNAL_API_TOKEN）
+JWT_SECRET=xxx INTERNAL_API_TOKEN=yyy go run main.go
```
## API
-| 方法 | 路径 | 说明 |
-|------|------|------|
-| GET | /healthz | 健康检查 |
-| GET | /ws?token=JWT | WebSocket 升级端点（JWT 鉴权） |
-| POST | /internal/push | 内部推送 API（Msg 服务调用） |
+| 方法 | 路径 | 鉴权 | 说明 |
+| ---- | --------------------------- | ------------------ | ------------------------------------------------ |
+| GET | `/ws?token=JWT` | JWT RS256 | WebSocket 升级（亦支持 `Authorization: Bearer`） |
+| POST | `/internal/push` | `X-Internal-Token` | 定向推送（msg 服务调用） |
+| POST | `/internal/broadcast` | `X-Internal-Token` | 广播 |
+| GET | `/internal/online/` | `X-Internal-Token` | 查在线状态 |
+| GET | `/healthz` | 无 | liveness 探针 |
+| GET | `/readyz` | 无 | readiness 探针（软失败 — ISSUE-058） |
+| GET | `/metrics` | 无 | Prometheus 指标 |
## 环境变量
-| 变量 | 默认值 | 说明 |
-|------|--------|------|
-| PUSH_GATEWAY_PORT | 8081 | 服务端口 |
-| JWT_SECRET | p1-dev-secret-change-in-production | JWT 密钥 |
+| 变量 | 默认值 | 说明 |
+| ----------------------------- | ---------------------------------------------- | ---------------------------------------- |
+| `PUSH_GATEWAY_PORT` | 8081 | HTTP 监听端口 |
+| `DEV_MODE` | false | 开发模式（跳过内部鉴权，接受 dev-token） |
+| `JWT_SECRET` | （DevMode 有默认） | JWT 密钥（生产必填） |
+| `JWKS_URL` | `http://localhost:50052/.well-known/jwks.json` | iam JWKS 端点 |
+| `INTERNAL_API_TOKEN` | （生产必填） | `/internal/*` 鉴权 token（ISSUE-002） |
+| `REDIS_URL` | `redis://localhost:6379/0` | Redis 连接 URL |
+| `KAFKA_BROKERS` | `localhost:9092` | Kafka broker 列表（逗号分隔） |
+| `KAFKA_NOTIFICATION_TOPIC` | `edu.notification.requested` | 消费 topic（ISSUE-053） |
+| `KAFKA_CONSUMER_GROUP` | `push-gateway` | consumer group |
+| `WS_ALLOWED_ORIGINS` | （空） | WebSocket Origin 白名单（逗号分隔） |
+| `MAX_CONNS_PER_USER` | 5 | 单用户最大连接数 |
+| `HEARTBEAT_INTERVAL_SECONDS` | 30 | 心跳间隔 |
+| `INSTANCE_ID` | （hostname） | 实例标识（Redis SET 成员） |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | `localhost:4318` | OTLP 端点 |
-## WebSocket 心跳
+## WebSocket 心跳（RFC 6455 控制帧）
-客户端发送 `ping` 文本帧，服务端回复 `pong`。
+- 客户端每 30s 发送 WebSocket **Ping 控制帧**（非文本消息）
+- 服务端自动回 Pong（gorilla/websocket 默认行为）
+- 服务端 `SetReadDeadline(60s)`：60s 未收到任何帧则断开
+- 每次 Pong 刷新 Redis 在线 SET 的 TTL（60s）
+
+## 内部包结构
+
+```
+internal/
+├─ config/ # 环境变量配置（shared-go/env）
+├─ hub/ # 连接池（userID -> connID -> Connection）
+├─ ws/ # WebSocket 升级 + /internal/* HTTP API
+├─ redisclient/ # Redis Pub/Sub + 在线 SET（ISSUE-058 启动重建）
+├─ kafkaconsumer/ # Kafka 消费（edu.notification.requested）
+├─ observability/ # slog logger + OTel tracer + Prometheus metrics
+└─ health/ # /healthz + /readyz（软失败）
+```
+
+## 优雅关闭顺序
+
+1. 收到 SIGTERM → 标记 Hub `closing`（拒绝新连接，`/readyz` 返 503）
+2. `Hub.CloseAll()`：向所有在线连接发送 close 帧（code=1001 going away）
+3. `srv.Shutdown(10s)`：停止 HTTP 服务，drain 在途请求
+4. Kafka consumer cancel + reader.Close
+5. Redis Pub/Sub cancel + client.Close
+6. Tracer shutdown（flush span）
+
+## 关键设计决策
+
+- **软失败**（ISSUE-058/006）：Redis/Kafka 故障时 `/readyz` 返 200 + `degraded:true`，不断本地 WebSocket
+- **Redis SET 启动重建**（ISSUE-058）：启动时 SREM 旧 instanceID，基于内存重建
+- **无 gRPC**（ISSUE-007）：仅 HTTP + WebSocket
+- **shared-go 复用**（ARB-015 §17.5）：tracer/logger/env/jwks 不重复实现
+
+完整 ADR 见 [02 架构设计 §14](./docs/02-architecture-design.md#14-架构决策记录adr)。
diff --git a/services/push-gateway/docs/02-architecture-design.md b/services/push-gateway/docs/02-architecture-design.md
index 0dc6742..639befa 100644
--- a/services/push-gateway/docs/02-architecture-design.md
+++ b/services/push-gateway/docs/02-architecture-design.md
@@ -1,11 +1,10 @@
# 模块架构设计文档 — push-gateway
-> AI：ai01（Go 网关层）
-> 阶段：阶段 2 交付物
-> 日期：2026-07-09
-> 关联：[01 理解确认书](./01-understanding.md)、[004 架构影响地图](../../../docs/architecture/004_architecture_impact_map.md)
+> 阶段：阶段 2 交付物（v1.0 仲裁回写于 2026-07-10）
+> 日期：2026-07-09（v0.1）/ 2026-07-10（v1.0）
+> 关联：[01 理解确认书](./01-understanding.md)、[004 架构影响地图](../../../docs/architecture/004_architecture_impact_map.md)、[president-final-rulings](../../../docs/architecture/president-final-rulings.md)
>
-> 本文档覆盖 ai-allocation §5 设计重点：WebSocket 连接生命周期、与 msg 的 gRPC 推送通道协议、用户 session 映射、水平扩展方案（Redis Pub/Sub 跨实例广播）。
+> 本文档覆盖：WebSocket 连接生命周期、与 msg 的 HTTP 推送通道协议、用户 session 映射、水平扩展方案（Redis Pub/Sub 跨实例广播）。仲裁结果见 §14 ADR。
---
@@ -53,7 +52,7 @@ graph TB
end
subgraph Kafka["Kafka 消费者（可选）"]
- K[Consumer Group
topic: edu.notification.events]
+ K[Consumer Group
topic: edu.notification.requested]
end
subgraph Msg["msg 服务"]
@@ -189,10 +188,10 @@ type Hub struct {
### 4.3 健康检查
-| 端点 | 检查逻辑 |
-| ---------- | ------------------------------------------------------------------------ |
-| `/healthz` | 进程存活，返回 200 |
-| `/readyz` | 检查 Redis 连接（PING）+ 在线连接数 > 0 时认为就绪；Redis 不可达返回 503 |
+| 端点 | 检查逻辑 |
+| ---------- | -------------------------------------------------------------------------------------------------------------- |
+| `/healthz` | 进程存活，返回 200 |
+| `/readyz` | Redis PING + Kafka 元数据探活；**软失败**（ISSUE-058/006）：失败返 200 + `degraded: true`，仅 Hub 关闭时返 503 |
### 4.4 指标端点
@@ -204,9 +203,11 @@ type Hub struct {
### 5.1 我消费的 Kafka 事件
-| Topic | Message | 触发场景 | 消费动作 |
-| ------------------------- | ----------------------- | -------------------------- | --------------------------------------------------------------------- |
-| `edu.notification.events` | `NotificationRequested` | msg 服务收到通知请求后发布 | 消费 → 调用 Hub.SendToUser 投递 → 若离线则 ACK 不重投（msg 服务落库） |
+| Topic | Message | 触发场景 | 消费动作 |
+| ---------------------------- | ----------------------- | -------------------------- | --------------------------------------------------------------------- |
+| `edu.notification.requested` | `NotificationRequested` | msg 服务收到通知请求后发布 | 消费 → 调用 Hub.SendToUser 投递 → 若离线则 ACK 不重投（msg 服务落库） |
+
+> **Topic 命名变更（ISSUE-053 / president §1.5）**：原名 `edu.notification.events` 不符合 G16 规则（`edu...`），已改为 `edu.notification.requested`。旧名作废。
**消费语义**：
@@ -298,13 +299,14 @@ func InitLogger(level string) {
| 指标名 | 类型 | 标签 | 描述 |
| ------------------------------------------- | --------- | ---------------- | ------------------------------------------------- |
| `push_gateway_active_connections` | Gauge | — | 当前在线连接数 |
-| `push_gateway_connections_per_user` | Gauge | — | 每用户连接数分布（仅暴露均值/最大） |
+| `push_gateway_connections_per_user_max` | Gauge | — | 单用户最大连接数（运行期峰值） |
| `push_gateway_messages_pushed_total` | Counter | event, result | 推送消息总数（result=delivered/dropped/offline） |
| `push_gateway_messages_dropped_total` | Counter | reason | 丢弃消息数（reason=channel_full/buffer_overflow） |
| `push_gateway_heartbeat_total` | Counter | — | 心跳接收总数 |
-| `push_gateway_disconnect_total` | Counter | reason | 断开连接数（reason=idle/error/closed） |
+| `push_gateway_disconnect_total` | Counter | reason | 断开连接数（reason=idle/error/closed/too_many） |
| `push_gateway_redis_pubsub_latency_seconds` | Histogram | direction | Pub/Sub 延迟（publish/subscribe） |
| `push_gateway_kafka_consumed_total` | Counter | topic, partition | Kafka 消费数 |
+| `push_gateway_redis_set_rebuild_total` | Counter | — | Redis online SET 重建次数（ISSUE-058 启动重建） |
| `go_*` | — | — | prom-client 默认 Go runtime 指标 |
### 6.5 Tracer
@@ -329,19 +331,31 @@ func Healthz(c *gin.Context) {
}
```
-### 6.7 /readyz 检查逻辑
+### 6.7 /readyz 检查逻辑（软失败 — ISSUE-058/006）
```go
-func Readyz(redisClient *redis.Client) gin.HandlerFunc {
- return func(c *gin.Context) {
- ctx, cancel := context.WithTimeout(c.Request.Context(), 1*time.Second)
- defer cancel()
- if err := redisClient.Ping(ctx).Err(); err != nil {
- c.JSON(503, gin.H{"status":"error","error":"redis unreachable"})
- return
- }
- c.JSON(200, gin.H{"status":"ok"})
+// internal/health/readyz.go
+// Redis/Kafka 不可达时不返 503，而是 200 + degraded:true，
+// 避免 K8s 在依赖短暂抖动时驱逐 Pod（ARB-015 §17.4）。
+// 仅 Hub.CloseAll 触发的 closing 状态返 503（优雅关闭期间）。
+func (rz *Readyzer) Handler(c *gin.Context) {
+ if rz.hub.IsClosing() {
+ c.JSON(503, readyzResponse{Status: "shutting_down", Degraded: true, ...})
+ return
}
+ deps := map[string]*dependencyStatus{}
+ degraded := false
+ // Redis 软探活
+ if err := rz.redis.Ping(ctx); err != nil {
+ deps["redis"] = &dependencyStatus{Ok: false, Error: err.Error()}
+ degraded = true
+ }
+ // Kafka 软探活
+ if err := rz.kafka.HealthCheck(ctx); err != nil {
+ deps["kafka"] = &dependencyStatus{Ok: false, Error: err.Error()}
+ degraded = true
+ }
+ c.JSON(200, readyzResponse{Status: statusText(degraded), Degraded: degraded, ...})
}
```
@@ -502,17 +516,17 @@ graph TB
## 9. 与其他模块的交互点（契约清单）
-| 方向 | 对方服务 | 协议 | 接口/事件 | 用途 |
-| ------ | ---------- | --------- | --------------------------------------------------- | --------------------- |
-| 被调用 | msg | HTTP | `POST /internal/push` | 定向推送 |
-| 被调用 | msg | HTTP | `POST /internal/broadcast` | 广播 |
-| 被调用 | msg | HTTP | `GET /internal/online/` | 查在线状态 |
-| 消费 | msg | Kafka | `edu.notification.events` / `NotificationRequested` | 异步广播通知 |
-| 调用 | iam | HTTP | `GET /.well-known/jwks.json` | RS256 公钥（P2） |
-| 调用 | Redis | Redis | Pub/Sub + SET | 跨实例广播 + 在线状态 |
-| 被调用 | 微前端 | WebSocket | `/ws` | 长连接 |
-| 被调用 | K8s/Docker | HTTP | `/healthz` `/readyz` | 探针 |
-| 被调用 | Prometheus | HTTP | `GET /metrics` | 指标采集 |
+| 方向 | 对方服务 | 协议 | 接口/事件 | 用途 |
+| ------ | ---------- | --------- | ------------------------------------------------------ | -------------------------------- |
+| 被调用 | msg | HTTP | `POST /internal/push` | 定向推送 |
+| 被调用 | msg | HTTP | `POST /internal/broadcast` | 广播 |
+| 被调用 | msg | HTTP | `GET /internal/online/` | 查在线状态 |
+| 消费 | msg | Kafka | `edu.notification.requested` / `NotificationRequested` | 异步广播通知（ISSUE-053 重命名） |
+| 调用 | iam | HTTP | `GET /.well-known/jwks.json` | RS256 公钥（P2） |
+| 调用 | Redis | Redis | Pub/Sub + SET | 跨实例广播 + 在线状态 |
+| 被调用 | 微前端 | WebSocket | `/ws` | 长连接 |
+| 被调用 | K8s/Docker | HTTP | `/healthz` `/readyz` | 探针 |
+| 被调用 | Prometheus | HTTP | `GET /metrics` | 指标采集 |
## 10. 风险与假设
@@ -571,3 +585,223 @@ graph TB
| `config/env.go` | 环境变量加载工具 | 两服务各有 getEnv，可统一 |
**提交方式**：通过 `# proto-change` 渠道向 coord 声明需求，coord 在 `shared-go` 中建立，ai01 在本服务中改为 import。
+
+---
+
+## 14. 架构决策记录（ADR）
+
+> 本节沉淀所有 ISSUE 仲裁结果与设计决策，作为未来变更的决策源。每条 ADR 引用对应的 ISSUE / president ruling / ARB 编号。
+
+### ADR-001 仅 WebSocket，不支持 SSE（ISSUE-001）
+
+- **决策**：push-gateway 仅实现 WebSocket 长连接通道，不提供 SSE（Server-Sent Events）端点。
+- **理由**：WebSocket 双向通信能力更强（服务端可主动推送 + 客户端可发 ack/heartbeat），单协议栈降低运维成本；SSE 单向推送且浏览器连接数受限（HTTP/1.1 下 6 个/域名）。微前端已统一使用 WebSocket 客户端。
+- **影响**：不支持仅支持 SSE 的旧客户端；客户端必须实现 WebSocket Ping 心跳。
+- **状态**：accepted（president-final-rulings §1.1）
+
+### ADR-002 X-Internal-Token 命名（ISSUE-002 / ARB-015 §17.3）
+
+- **决策**：`/internal/*` API 鉴权头从 `X-Internal-Key` 改名为 `X-Internal-Token`，环境变量从 `INTERNAL_API_KEY` 改为 `INTERNAL_API_TOKEN`。
+- **理由**：与项目其他服务（api-gateway、msg）的内部鉴权头命名保持一致；"Token" 更准确表达 bearer 语义。
+- **影响**：msg 服务调用方需同步修改请求头；DevMode 仍跳过校验。
+- **状态**：accepted（president §7.2）
+
+### ADR-003 单实例容量 10w+ 连接（ISSUE-003）
+
+- **决策**：单实例目标连接数从 50k 提升到 10w+，通过 goroutine-per-connection + send chan(cap 64) + 内存优化实现。
+- **理由**：10w 连接估算内存 ~1.5GB（每连接 ~15KB：goroutine 栈 8KB + chan 64×1.5KB + websocket buffer 4KB），Go runtime 可承载；实际峰值按 80% 水位（8w）规划副本数。
+- **影响**：单 Pod resources.limits.memory 建议 2GB；超限时水平扩容而非单实例加内存。
+- **状态**：accepted（president §3.1）
+
+### ADR-004 无 DB，审计由 msg 服务负责（ISSUE-005）
+
+- **决策**：push-gateway 不持久化任何业务数据（无 DB），消息审计、离线落库、重试队列均由 msg 服务负责。
+- **理由**：push-gateway 职责单一（连接管理 + 消息转发），引入 DB 会增加状态一致性复杂度；msg 服务已有 Outbox + DB，天然承担审计职责。
+- **影响**：push-gateway 崩溃时在途消息丢失（msg 通过 Outbox 重投兜底）；`/internal/push` 同步响应返 `delivered/online` 供 msg 决策是否离线推送。
+- **状态**：accepted（president §5.1）
+
+### ADR-005 Kafka topic 命名（ISSUE-053）
+
+- **决策**：消费的 Kafka topic 从 `edu.notification.events` 改为 `edu.notification.requested`。
+- **理由**：符合 G16 命名规则 `edu...`，`requested` 表达"通知请求"语义，`events` 过于宽泛。
+- **影响**：msg 服务 Outbox TOPIC_MAP 需同步修改；CI 检测 breaking change。
+- **状态**：accepted（president-final-rulings §1.5）
+
+### ADR-006 /readyz 软失败（ISSUE-058 / ISSUE-006 / ARB-015 §17.4）
+
+- **决策**：`/readyz` 探针采用软失败语义：Redis 或 Kafka 不可达时返回 HTTP 200 + `degraded: true`，而非 503。
+- **理由**：K8s readinessProbe 返 503 会立即从 Service endpoints 驱逐 Pod，导致 Redis 短暂抖动时所有 push-gateway 实例被驱逐、WebSocket 连接全断。软失败让 Pod 留在 endpoints 内，仅降级（无跨实例 fanout / 无 Kafka 消费），抖动恢复后自动恢复。
+- **例外**：Hub 处于 `closing` 状态时返 503（优雅关闭期间确实不应接收新连接）。
+- **影响**：msg 服务调用 `/internal/push` 时需容忍 `delivered:false, online:false`（Redis 不可达时的保守响应）；K8s readinessProbe 配置 `failureThreshold: 5` 给恢复留窗口。
+- **状态**：accepted（president-final-rulings §1.6 / ARB-015 §17.4）
+
+### ADR-007 Redis online SET 启动重建（ISSUE-058）
+
+- **决策**：push-gateway 实例启动时主动扫描 `edu:push:online:*` 并 SREM 自己的 instanceID，然后基于本地 Hub 内存状态重新 SADD。
+- **理由**：实例崩溃重启后，旧 instanceID 在 Redis SET 中可能尚未过期（60s TTL 窗口），直接 SADD 同名 instanceID 是 no-op，无法区分"旧连接已失效"与"新连接已建立"。先清后建确保 SET 与内存一致。
+- **不一致窗口**：崩溃到 SET 过期之间（最长 60s），`/internal/online` 可能误报 online:true；msg 通过 `delivered:false` 重试或离线推送兜底。
+- **指标**：`push_gateway_redis_set_rebuild_total` 记录重建次数。
+- **状态**：accepted（president-final-rulings §1.6）
+
+### ADR-008 Redis 作为必需依赖但软失败（ISSUE-055 vs ISSUE-058 仲裁）
+
+- **决策**：Redis 是 push-gateway 的必需依赖（部署前置条件），但运行期 Redis 故障采用软失败（ADR-006），不阻断 WebSocket 本地投递。
+- **理由**：ISSUE-055 主张 Redis 不可达时 push-gateway 应 503 停服；ISSUE-058 主张软失败。ARB-015 §17.4 裁定 ISSUE-058 优先：Redis 故障时本地 WebSocket 连接仍可服务，仅跨实例 fanout 与在线状态查询降级。
+- **部署约束**：启动时 Redis 不可达记为 degraded 并继续启动（不 Fatal）；生产环境应通过 K8s deployment `initContainer` 等待 Redis 就绪。
+- **状态**：accepted（ARB-015 §17.4）
+
+### ADR-009 纯 HTTP + WebSocket，无 gRPC（ISSUE-007）
+
+- **决策**：push-gateway 对外仅暴露 HTTP（`/internal/*` API）+ WebSocket（`/ws`），不实现 gRPC 服务。
+- **理由**：msg 服务调用 push-gateway 用 HTTP 已足够（同步请求-响应语义）；引入 gRPC 会增加 proto 契约维护成本与 buf 生成链路。gRPC 的流式推送能力由 WebSocket 承担。
+- **影响**：msg 服务无需生成 push-gateway 的 gRPC client；`packages/shared-proto/proto/` 无 push-gateway.proto。
+- **状态**：accepted（ARB-015 §17.6）
+
+### ADR-010 shared-go 复用（ARB-015 §17.5）
+
+- **决策**：tracer / logger / env / jwks 四个横切模块统一从 `packages/shared-go/` 导入，不在 push-gateway 内重复实现。
+- **理由**：与 api-gateway 共用同一套 OTel SDK 配置、slog 日志格式、JWKS 缓存逻辑，避免代码漂移。
+- **影响**：`go.mod` 依赖 `github.com/edu-cloud/shared-go`（replace 到 `../../packages/shared-go`）；`internal/observability/tracer.go` 仅包装 `shared-go/tracer.Init`。
+- **状态**：accepted（ARB-015 §17.5）
+
+---
+
+## 15. 非功能性需求（NFR）
+
+### 15.1 性能
+
+| 指标 | 目标 | 测量方式 | 验收标准 |
+| ------------------ | ------- | ------------------------------------ | ----------------------------- |
+| 单实例最大连接数 | 10w+ | `push_gateway_active_connections` | 持续 30min 稳定 ≥ 10w 不崩 |
+| 本实例推送延迟 P99 | < 50ms | `push.message` span | 压测 10w 连接下 P99 < 50ms |
+| 跨实例推送延迟 P99 | < 200ms | Redis Pub/Sub round-trip | 3 实例 10w 连接下 P99 < 200ms |
+| 广播推送延迟 P99 | < 100ms | `push_gateway_messages_pushed_total` | 10w 连接广播 P99 < 100ms |
+| 心跳间隔 | 30s | 客户端配置 | 客户端 Ping 间隔 30s ± 5s |
+| 空闲超时 | 60s | `SetReadDeadline` | 60s 无帧断开 |
+| 消息体上限 | 64KB | `SetReadLimit` | 超限返回 1009 |
+
+### 15.2 可用性
+
+| 指标 | 目标 | 实现方式 |
+| -------------- | ------ | --------------------------------------------------- |
+| 服务可用性 | 99.9% | 多副本部署 + K8s 自愈 + /readyz 软失败 |
+| Redis 故障降级 | 不断连 | 软失败保留本地 WebSocket 服务（ADR-006/008） |
+| Kafka 故障降级 | 不断连 | 软失败保留 HTTP `/internal/push` 通道 |
+| 滚动升级零中断 | 是 | Hub.CloseAll + srv.Shutdown + 10s drain |
+| 单实例崩溃恢复 | < 60s | K8s 重启 + Redis SET TTL 过期 + 启动重建（ADR-007） |
+
+### 15.3 安全
+
+| 需求 | 实现 |
+| -------------- | -------------------------------------------------------------- |
+| WebSocket 鉴权 | JWT RS256（iam JWKS，5min 缓存）；DevMode 接受 `dev-token` |
+| 内部 API 鉴权 | `X-Internal-Token` 头（K8s Secret 注入，与 msg 共享） |
+| Origin 白名单 | `WS_ALLOWED_ORIGINS` 环境变量，DevMode 空白名单允许所有 |
+| 非 root 运行 | Dockerfile `USER app` (uid 1001) |
+| 最小镜像 | alpine:3.20 + CGO_ENABLED=0 静态编译 |
+| 无密钥泄露 | INTERNAL_API_TOKEN / JWT_SECRET 通过 K8s Secret 注入，不入镜像 |
+
+### 15.4 可观测性
+
+| 支柱 | 实现 |
+| -------- | ----------------------------------------------------------------------- |
+| 日志 | slog JSON（prod）/ text（dev），字段 `service/user_id/conn_id/trace_id` |
+| 指标 | 9 个 Prometheus 指标 + Go runtime 默认指标，`/metrics` 端点 |
+| 追踪 | OTel SDK via shared-go/tracer，otelgin 中间件自动埋点 HTTP span |
+| 健康检查 | `/healthz`（liveness）+ `/readyz`（readiness，软失败） |
+
+---
+
+## 16. 失败模式与恢复
+
+### 16.1 失败模式矩阵
+
+| 失败场景 | 检测方式 | 系统行为 | 恢复方式 |
+| ---------------------- | ---------------------- | -------------------------------------------------------------- | -------------------------------- |
+| Redis 不可达 | `/readyz` 软探活 | 200 + `degraded:true`；跨实例 fanout 停止；本地 WebSocket 继续 | Redis 恢复后自动重连 |
+| Kafka 不可达 | `/readyz` 软探活 | 200 + `degraded:true`；HTTP `/internal/push` 通道继续 | Kafka 恢复后 consumer 自动重连 |
+| iam JWKS 不可达 | jwks.Fetcher 返错 | 新 WebSocket 连接被拒（401）；已建连接不受影响 | JWKS 缓存 5min，恢复后新连接可用 |
+| 单实例崩溃 | K8s livenessProbe 失败 | K8s 重启 Pod；该实例连接断开，客户端重连到其他实例 | Redis SET 60s 过期 + 启动重建 |
+| 客户端网络抖动 | `SetReadDeadline(60s)` | 60s 无帧断开连接；客户端重连 | 客户端指数退避重连 |
+| 单用户连接超限 | Hub.counters 检查 | 拒绝新连接，返 close 帧 1008 | 客户端关闭旧连接后重连 |
+| send channel 满 | `Send()` 返 false | 丢弃消息，`messages_dropped_total{reason=channel_full}` +1 | 消费端 drain 后恢复 |
+| Kafka 消费失败 | 3 次重试后 | 发送 DLQ（`edu.notification.requested.dlq`），commit 跳过 | 人工排查 DLQ |
+| Redis Pub/Sub 消息丢失 | fire-and-forget 特性 | msg 服务通过 `delivered:false` 响应触发重试或离线推送 | msg Outbox 重投 |
+| 内存 OOM | K8s OOMKilled | Pod 重启，连接断开，客户端重连 | 调整 resources.limits.memory |
+
+### 16.2 降级矩阵
+
+| 依赖故障 | 本地 WebSocket | HTTP /internal/push | Kafka 消费 | 跨实例 fanout | /readyz 状态 |
+| ------------- | -------------- | ------------------- | ---------- | ------------- | -------------- |
+| Redis 故障 | ✅ 正常 | ✅ 本地投递 | ✅ 正常 | ❌ 停止 | degraded |
+| Kafka 故障 | ✅ 正常 | ✅ 正常 | ❌ 停止 | ✅ 正常 | degraded |
+| iam JWKS 故障 | ⚠️ 新连接被拒 | ✅ 正常 | ✅ 正常 | ✅ 正常 | ok（缓存可用） |
+| Hub 关闭中 | ❌ 拒绝新连接 | ❌ 503 | ⏸️ drain | ❌ 停止 | shutting_down |
+
+---
+
+## 17. 容量估算
+
+### 17.1 单实例资源估算
+
+| 资源 | 估算公式 | 10w 连接目标值 | 备注 |
+| ------------ | ------------------------------------- | -------------- | ----------------------------- |
+| 内存 | 每连接 ~15KB × 10w + Go runtime 200MB | ~1.7GB | goroutine 栈 + chan + buffer |
+| goroutine 数 | 每连接 2（reader+writer）+ 10（后台） | ~20w | Go runtime 调度开销可接受 |
+| 文件描述符 | 每连接 1（WebSocket fd）+ 50（其他） | ~10w | ulimit -n 需调到 200000 |
+| CPU | 心跳处理 10w/30s = 3.3k QPS | 1-2 core | 心跳轻量，主要 CPU 在消息推送 |
+
+### 17.2 K8s Pod 资源配置建议
+
+```yaml
+resources:
+ requests:
+ cpu: 500m
+ memory: 2Gi
+ limits:
+ cpu: 2000m
+ memory: 3Gi # 留 1.3GB 余量应对突发
+```
+
+### 17.3 副本数规划
+
+| 场景 | 在线用户数 | 单实例容量 | 副本数 | 总内存 |
+| -------------- | ---------- | ---------- | ------ | ------ |
+| 开发环境 | 100 | 10w | 1 | 3GB |
+| 测试环境 | 1k | 10w | 1 | 3GB |
+| 预发环境 | 1w | 10w | 2 | 6GB |
+| 生产（小规模） | 5w | 10w | 2 | 6GB |
+| 生产（大规模） | 20w | 10w | 3 | 9GB |
+| 生产（峰值） | 50w | 10w | 6 | 18GB |
+
+> **副本数公式**：`ceil(在线用户数 × 0.8 / 10w) + 1`（80% 水位 + 1 冗余）
+
+### 17.4 Redis 资源估算
+
+| 数据结构 | 单条大小 | 10w 在线用户总量 | 备注 |
+| ------------------------------------ | -------- | ---------------- | ------------------------- |
+| `edu:push:online:` | ~50B | ~5MB | SET + TTL |
+| `edu:push:session::` | ~100B | ~10MB | HASH，每连接一条 |
+| Pub/Sub channel 消息 | ~200B | 瞬时 | 不持久化 |
+| `edu:push:idempotent:` | ~30B | ~3MB/h | 24h TTL，按 10w 事件/h 估 |
+
+> Redis 总内存需求 < 50MB（10w 在线用户），现有 `edu-redis` 容器 128m 限制足够。
+
+### 17.5 Kafka 容量估算
+
+| 指标 | 估算 | 备注 |
+| ------------ | ----------------- | ---------------------------- |
+| 消息速率 | 1k 通知/s（峰值） | 考试开始、成绩发布等批量场景 |
+| 消息大小 | ~500B | NotificationRequested JSON |
+| 吞吐量 | ~500KB/s | 单 partition 足够 |
+| Partition 数 | 3 | 容错 + 并行消费 |
+| 保留时间 | 24h | 失败重试窗口 |
+
+---
+
+## 18. 变更日志
+
+| 日期 | 版本 | 变更 | 来源 |
+| ---------- | ---- | --------------------------------------------------------------------------------------------------------- | --------------------------------- |
+| 2026-07-09 | 0.1 | 初始架构设计 | 阶段 2 交付 |
+| 2026-07-10 | 1.0 | 仲裁回写：ISSUE-001/002/003/005/053/055/056/058/007；新增 §14 ADR / §15 NFR / §16 失败模式 / §17 容量估算 | ARB-015 + president-final-rulings |
diff --git a/services/push-gateway/go.mod b/services/push-gateway/go.mod
index ec9341a..102edaf 100644
--- a/services/push-gateway/go.mod
+++ b/services/push-gateway/go.mod
@@ -12,10 +12,7 @@ require (
github.com/redis/go-redis/v9 v9.7.0
github.com/segmentio/kafka-go v0.4.48
go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.69.0
- go.opentelemetry.io/otel v1.44.0
- go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0
- go.opentelemetry.io/otel/sdk v1.44.0
- go.uber.org/zap v1.27.0
+ go.opentelemetry.io/otel/trace v1.44.0
)
require (
@@ -38,6 +35,7 @@ require (
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
+ github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
@@ -45,6 +43,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.3.1 // indirect
+ github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
@@ -54,9 +53,11 @@ require (
github.com/ugorji/go/codec v1.3.1 // indirect
go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+ go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
- go.opentelemetry.io/otel/trace v1.44.0 // indirect
+ go.opentelemetry.io/otel/sdk v1.44.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/arch v0.27.0 // indirect
diff --git a/services/push-gateway/go.sum b/services/push-gateway/go.sum
index 94a0f02..9136b80 100644
--- a/services/push-gateway/go.sum
+++ b/services/push-gateway/go.sum
@@ -1,5 +1,9 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/bytedance/gopkg v0.1.4 h1:oZnQwnX82KAIWb7033bEwtxvTqXcYMxDBaQxo5JJHWM=
github.com/bytedance/gopkg v0.1.4/go.mod h1:v1zWfPm21Fb+OsyXN2VAHdL6TBb2L88anLQgdyje6R4=
github.com/bytedance/sonic v1.15.1 h1:nJD5PmM0vY7J8CT6MxoqbVAAMhkSmV2HgRAUrrpLoOw=
@@ -15,6 +19,8 @@ github.com/cloudwego/base64x v0.1.7/go.mod h1:Cu1PV9zfrSf7ET2tIbWbbEy7jO7HHJ13q4
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/gin-contrib/sse v1.1.1 h1:uGYpNwTacv5R68bSGMapo62iLTRa9l5zxGCps4hK6ko=
@@ -53,6 +59,7 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF2
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
@@ -76,6 +83,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc=
github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
@@ -90,8 +99,12 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic=
github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs=
+github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -107,6 +120,14 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY=
github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8=
go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
@@ -141,14 +162,53 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU=
golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8=
diff --git a/services/push-gateway/internal/config/config_test.go b/services/push-gateway/internal/config/config_test.go
new file mode 100644
index 0000000..ecff8f2
--- /dev/null
+++ b/services/push-gateway/internal/config/config_test.go
@@ -0,0 +1,120 @@
+package config
+
+import (
+ "os"
+ "testing"
+)
+
+// TestLoadDevMode verifies DevMode defaults and that required vars are optional.
+func TestLoadDevMode(t *testing.T) {
+ t.Setenv("DEV_MODE", "true")
+ t.Setenv("JWT_SECRET", "")
+ t.Setenv("INTERNAL_API_TOKEN", "")
+
+ cfg := Load()
+
+ if !cfg.DevMode {
+ t.Error("DevMode = false, want true")
+ }
+ if cfg.JWTSecret != devJWTSecret {
+ t.Errorf("JWTSecret = %q, want dev default %q", cfg.JWTSecret, devJWTSecret)
+ }
+ if cfg.InternalAPIToken != "" {
+ t.Errorf("InternalAPIToken = %q, want empty in DevMode", cfg.InternalAPIToken)
+ }
+ if cfg.Port != "8081" {
+ t.Errorf("Port = %q, want 8081", cfg.Port)
+ }
+ if cfg.MaxConnsPerUser != 5 {
+ t.Errorf("MaxConnsPerUser = %d, want 5", cfg.MaxConnsPerUser)
+ }
+ if cfg.HeartbeatInterval != 30 {
+ t.Errorf("HeartbeatInterval = %d, want 30", cfg.HeartbeatInterval)
+ }
+ if cfg.KafkaNotificationTopic != "edu.notification.requested" {
+ t.Errorf("KafkaNotificationTopic = %q, want edu.notification.requested", cfg.KafkaNotificationTopic)
+ }
+ if cfg.KafkaConsumerGroup != "push-gateway" {
+ t.Errorf("KafkaConsumerGroup = %q, want push-gateway", cfg.KafkaConsumerGroup)
+ }
+ if cfg.InstanceID == "" {
+ t.Error("InstanceID should default to hostname, got empty")
+ }
+}
+
+// TestLoadProdMode verifies production requires JWT_SECRET + INTERNAL_API_TOKEN.
+func TestLoadProdMode(t *testing.T) {
+ // Ensure the env vars are unset so Load fails. We can't call os.Exit in
+ // a test, so we verify the required behavior by setting DEV_MODE=false
+ // and checking that Load triggers log.Fatal. Since log.Fatal calls
+ // os.Exit, we instead test the happy path: all required vars set.
+ t.Setenv("DEV_MODE", "false")
+ t.Setenv("JWT_SECRET", "prod-secret")
+ t.Setenv("INTERNAL_API_TOKEN", "prod-token")
+
+ cfg := Load()
+
+ if cfg.DevMode {
+ t.Error("DevMode = true, want false")
+ }
+ if cfg.JWTSecret != "prod-secret" {
+ t.Errorf("JWTSecret = %q, want prod-secret", cfg.JWTSecret)
+ }
+ if cfg.InternalAPIToken != "prod-token" {
+ t.Errorf("InternalAPIToken = %q, want prod-token", cfg.InternalAPIToken)
+ }
+}
+
+// TestParseOrigins verifies comma-separated origin parsing.
+func TestParseOrigins(t *testing.T) {
+ tests := []struct {
+ name string
+ raw string
+ want []string
+ }{
+ {"empty", "", nil},
+ {"single", "http://localhost:3000", []string{"http://localhost:3000"}},
+ {"multi", "http://a.com,http://b.com,http://c.com", []string{"http://a.com", "http://b.com", "http://c.com"}},
+ {"with-spaces", " http://a.com , http://b.com ", []string{"http://a.com", "http://b.com"}},
+ {"trailing-comma", "http://a.com,", []string{"http://a.com"}},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ got := parseOrigins(tc.raw)
+ if len(got) != len(tc.want) {
+ t.Errorf("parseOrigins(%q) = %v, want %v", tc.raw, got, tc.want)
+ return
+ }
+ for i := range got {
+ if got[i] != tc.want[i] {
+ t.Errorf("parseOrigins(%q)[%d] = %q, want %q", tc.raw, i, got[i], tc.want[i])
+ }
+ }
+ })
+ }
+}
+
+// TestParseBrokers verifies comma-separated broker parsing.
+func TestParseBrokers(t *testing.T) {
+ got := parseBrokers("kafka1:9092,kafka2:9092,kafka3:9092")
+ if len(got) != 3 {
+ t.Fatalf("parseBrokers returned %d items, want 3", len(got))
+ }
+ if got[0] != "kafka1:9092" || got[1] != "kafka2:9092" || got[2] != "kafka3:9092" {
+ t.Errorf("parseBrokers = %v", got)
+ }
+}
+
+// TestGenerateInstanceID verifies fallback to hostname.
+func TestGenerateInstanceID(t *testing.T) {
+ id := generateInstanceID()
+ if id == "" {
+ t.Error("generateInstanceID returned empty string")
+ }
+ host, err := os.Hostname()
+ if err == nil && host != "" {
+ if id != host {
+ t.Errorf("generateInstanceID = %q, want hostname %q", id, host)
+ }
+ }
+}
diff --git a/services/push-gateway/internal/health/readyz.go b/services/push-gateway/internal/health/readyz.go
new file mode 100644
index 0000000..0d640db
--- /dev/null
+++ b/services/push-gateway/internal/health/readyz.go
@@ -0,0 +1,186 @@
+// Package health implements the /healthz (liveness) and /readyz (readiness)
+// endpoints for push-gateway.
+//
+// /healthz is a trivial liveness probe: it returns 200 + {status:ok} as long
+// as the process is running and the Gin router is serving.
+//
+// /readyz reports readiness based on downstream dependency health. Per
+// ARB-015 §17.4 / ISSUE-058 the readiness probe uses SOFT FAILURE semantics:
+// when Redis or Kafka is unavailable the endpoint still returns HTTP 200 but
+// carries `degraded: true` plus a `dependencies` block describing which
+// component failed. This prevents Kubernetes from evicting the pod when a
+// transient dependency blip occurs, at the cost of accepting some degraded
+// behavior (no cross-instance fanout, no Kafka consumption) during the blip.
+// Only the Hub being in a shutting-down state returns a non-200 (503).
+//
+// The hard-failure case is limited to:
+// - Hub.closing == true (process is shutting down) -> 503
+// - Internal misconfiguration (both Redis and Kafka missing in non-DevMode)
+// -> still 200 + degraded, since the pod can still serve local WebSocket
+// traffic.
+package health
+
+import (
+ "context"
+ "net/http"
+ "time"
+
+ "github.com/edu-cloud/push-gateway/internal/hub"
+ "github.com/edu-cloud/push-gateway/internal/kafkaconsumer"
+ "github.com/edu-cloud/push-gateway/internal/redisclient"
+ "github.com/gin-gonic/gin"
+)
+
+// probeTimeout bounds each dependency check so a hung dependency cannot stall
+// the readiness probe. Kubernetes typically expects /readyz to respond within
+// 5s; we use 1s per probe to leave headroom.
+const probeTimeout = 1 * time.Second
+
+// dependencyStatus is the per-component health snapshot included in the
+// /readyz response body.
+type dependencyStatus struct {
+ Ok bool `json:"ok"`
+ Error string `json:"error,omitempty"`
+ Latency int64 `json:"latency_ms,omitempty"`
+}
+
+// readyzResponse is the ActionState-shaped envelope returned by /readyz.
+// `degraded` is true when at least one non-critical dependency is unhealthy.
+type readyzResponse struct {
+ Status string `json:"status"`
+ Service string `json:"service"`
+ InstanceID string `json:"instance_id"`
+ Degraded bool `json:"degraded"`
+ Connections int `json:"connections"`
+ Users int `json:"users"`
+ Dependencies map[string]*dependencyStatus `json:"dependencies"`
+}
+
+// Healthz returns a trivial liveness handler: 200 + {status:ok, service}.
+// Liveness never depends on downstream components — if the process can serve
+// the request, it is alive.
+func Healthz(service string) gin.HandlerFunc {
+ return func(c *gin.Context) {
+ c.JSON(http.StatusOK, gin.H{
+ "status": "ok",
+ "service": service,
+ })
+ }
+}
+
+// Readyzer builds the /readyz handler. It probes Redis (PING) and Kafka
+// (reader lag / connectivity) and reports degraded state per ARB-015 §17.4.
+//
+// The Hub is used to report local connection counts and to detect the
+// shutting-down state (which triggers a hard 503). The redisClient may be nil
+// in DevMode; the kafkaConsumer may be nil when KAFKA_BROKERS is unset. Both
+// nil cases are reported as degraded rather than failing the probe.
+type Readyzer struct {
+ hub *hub.Hub
+ redis *redisclient.Client
+ kafka *kafkaconsumer.Consumer
+ instance string
+ service string
+}
+
+// NewReadyzer constructs a Readyzer. redis and kafka may be nil; the resulting
+// probe will mark the missing dependency as degraded.
+func NewReadyzer(h *hub.Hub, r *redisclient.Client, k *kafkaconsumer.Consumer, service, instanceID string) *Readyzer {
+ return &Readyzer{
+ hub: h,
+ redis: r,
+ kafka: k,
+ instance: instanceID,
+ service: service,
+ }
+}
+
+// Handler is the gin.HandlerFunc for GET /readyz.
+func (rz *Readyzer) Handler(c *gin.Context) {
+ // Hard failure: Hub is shutting down — reject new traffic.
+ // (Hub.CloseAll sets closing=true; we treat this as 503 so the load
+ // balancer stops sending WebSocket upgrades during drain.)
+ if rz.hub.IsClosing() {
+ c.JSON(http.StatusServiceUnavailable, readyzResponse{
+ Status: "shutting_down",
+ Service: rz.service,
+ InstanceID: rz.instance,
+ Degraded: true,
+ Connections: rz.hub.ActiveConnections(),
+ Users: rz.hub.UserCount(),
+ Dependencies: map[string]*dependencyStatus{
+ "hub": {Ok: false, Error: "hub is closing"},
+ },
+ })
+ return
+ }
+
+ deps := make(map[string]*dependencyStatus, 2)
+ degraded := false
+
+ // Redis probe (soft failure).
+ if rz.redis == nil {
+ deps["redis"] = &dependencyStatus{Ok: false, Error: "not configured"}
+ degraded = true
+ } else {
+ deps["redis"] = probeRedis(rz.redis)
+ if !deps["redis"].Ok {
+ degraded = true
+ }
+ }
+
+ // Kafka probe (soft failure).
+ if rz.kafka == nil {
+ deps["kafka"] = &dependencyStatus{Ok: false, Error: "not configured"}
+ degraded = true
+ } else {
+ deps["kafka"] = probeKafka(rz.kafka)
+ if !deps["kafka"].Ok {
+ degraded = true
+ }
+ }
+
+ // Always 200 (unless shutting down) per ISSUE-058/006 soft-failure rule.
+ c.JSON(http.StatusOK, readyzResponse{
+ Status: statusText(degraded),
+ Service: rz.service,
+ InstanceID: rz.instance,
+ Degraded: degraded,
+ Connections: rz.hub.ActiveConnections(),
+ Users: rz.hub.UserCount(),
+ Dependencies: deps,
+ })
+}
+
+// statusText returns "ok" when healthy, "degraded" when at least one
+// dependency is unhealthy.
+func statusText(degraded bool) string {
+ if degraded {
+ return "degraded"
+ }
+ return "ok"
+}
+
+// probeRedis issues a PING with a short timeout and reports latency.
+func probeRedis(r *redisclient.Client) *dependencyStatus {
+ ctx, cancel := context.WithTimeout(context.Background(), probeTimeout)
+ defer cancel()
+ start := time.Now()
+ if err := r.Ping(ctx); err != nil {
+ return &dependencyStatus{Ok: false, Error: err.Error()}
+ }
+ return &dependencyStatus{Ok: true, Latency: time.Since(start).Milliseconds()}
+}
+
+// probeKafka checks that the consumer reader is still reachable. We use a
+// lightweight Lag() call (segmentio/kafka-go Client API); on failure the
+// consumer is marked degraded. Note: a degraded Kafka does NOT block WebSocket
+// traffic — it only pauses notification consumption until recovery.
+func probeKafka(k *kafkaconsumer.Consumer) *dependencyStatus {
+ ctx, cancel := context.WithTimeout(context.Background(), probeTimeout)
+ defer cancel()
+ if err := k.HealthCheck(ctx); err != nil {
+ return &dependencyStatus{Ok: false, Error: err.Error()}
+ }
+ return &dependencyStatus{Ok: true}
+}
diff --git a/services/push-gateway/internal/hub/hub.go b/services/push-gateway/internal/hub/hub.go
index f667a26..9153ad3 100644
--- a/services/push-gateway/internal/hub/hub.go
+++ b/services/push-gateway/internal/hub/hub.go
@@ -172,6 +172,15 @@ func (h *Hub) HasUser(userID string) bool {
return ok && len(conns) > 0
}
+// IsClosing reports whether CloseAll has been invoked. The /readyz probe uses
+// this to return 503 during graceful shutdown so the load balancer stops
+// sending new WebSocket upgrades while existing connections drain.
+func (h *Hub) IsClosing() bool {
+ h.mu.RLock()
+ defer h.mu.RUnlock()
+ return h.closing
+}
+
// Register adds a new connection for userID. Returns ErrTooManyConnections
// when the per-user cap is exceeded and ErrHubClosing when the Hub is shutting
// down. The returned Connection must be Unregistered by the caller on close.
@@ -262,7 +271,13 @@ func (h *Hub) SendToUser(userID string, message []byte) int {
// Cross-instance broadcast is handled separately via Redis Pub/Sub.
func (h *Hub) Broadcast(message []byte) int {
h.mu.RLock()
- snapshot := make([]*Connection, 0, h.ActiveConnections())
+ // Count inline: calling ActiveConnections() here would re-enter RLock
+ // while we already hold it, which can deadlock if a writer is waiting.
+ total := 0
+ for _, conns := range h.clients {
+ total += len(conns)
+ }
+ snapshot := make([]*Connection, 0, total)
for _, conns := range h.clients {
for _, c := range conns {
snapshot = append(snapshot, c)
@@ -285,7 +300,13 @@ func (h *Hub) Broadcast(message []byte) int {
func (h *Hub) CloseAll() {
h.mu.Lock()
h.closing = true
- all := make([]*Connection, 0, h.ActiveConnections())
+ // Count inline instead of calling ActiveConnections() to avoid reentrant
+ // RLock while holding the write lock (Go's sync.RWMutex is non-reentrant).
+ total := 0
+ for _, conns := range h.clients {
+ total += len(conns)
+ }
+ all := make([]*Connection, 0, total)
for _, conns := range h.clients {
for _, c := range conns {
all = append(all, c)
diff --git a/services/push-gateway/internal/hub/hub_test.go b/services/push-gateway/internal/hub/hub_test.go
new file mode 100644
index 0000000..d9e3527
--- /dev/null
+++ b/services/push-gateway/internal/hub/hub_test.go
@@ -0,0 +1,238 @@
+package hub
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/gorilla/websocket"
+)
+
+// newTestConn returns a real *websocket.Conn (client side) backed by a local
+// httptest server. The server-side handler blocks on a done channel until the
+// test ends; cleanup closes the client conn first, then the server, avoiding
+// the httptest.Server.Close() hang that occurs when the handler is blocked on
+// ReadMessage.
+func newTestConn(t *testing.T) *websocket.Conn {
+ t.Helper()
+ done := make(chan struct{})
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ up := websocket.Upgrader{}
+ c, err := up.Upgrade(w, r, nil)
+ if err != nil {
+ t.Errorf("upgrade: %v", err)
+ return
+ }
+ <-done
+ _ = c.Close()
+ }))
+ cli, _, err := websocket.DefaultDialer.Dial(strings.Replace(srv.URL, "http:", "ws:", 1), nil)
+ if err != nil {
+ close(done)
+ srv.Close()
+ t.Fatalf("dial: %v", err)
+ }
+ t.Cleanup(func() {
+ close(done)
+ _ = cli.Close()
+ srv.Close()
+ })
+ return cli
+}
+
+// TestRegisterAndUnregister verifies basic Hub register/unregister bookkeeping.
+func TestRegisterAndUnregister(t *testing.T) {
+ h := NewHub(5)
+ conn := newTestConn(t)
+
+ c, err := h.Register("user-1", conn)
+ if err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ if c.UserID != "user-1" {
+ t.Errorf("UserID = %q, want user-1", c.UserID)
+ }
+ if c.ConnID == "" {
+ t.Error("ConnID should be non-empty")
+ }
+ if h.ActiveConnections() != 1 {
+ t.Errorf("ActiveConnections = %d, want 1", h.ActiveConnections())
+ }
+ if h.UserCount() != 1 {
+ t.Errorf("UserCount = %d, want 1", h.UserCount())
+ }
+ if !h.HasUser("user-1") {
+ t.Error("HasUser(user-1) = false, want true")
+ }
+
+ h.Unregister(c)
+ if h.ActiveConnections() != 0 {
+ t.Errorf("after Unregister ActiveConnections = %d, want 0", h.ActiveConnections())
+ }
+ if h.HasUser("user-1") {
+ t.Error("HasUser(user-1) = true, want false")
+ }
+}
+
+// TestRegisterTooManyConnections verifies the per-user cap is enforced.
+func TestRegisterTooManyConnections(t *testing.T) {
+ h := NewHub(2)
+ for i := 0; i < 2; i++ {
+ conn := newTestConn(t)
+ if _, err := h.Register("user-cap", conn); err != nil {
+ t.Fatalf("Register #%d: %v", i, err)
+ }
+ }
+ conn := newTestConn(t)
+ _, err := h.Register("user-cap", conn)
+ if err != ErrTooManyConnections {
+ t.Errorf("Register over cap err = %v, want ErrTooManyConnections", err)
+ }
+}
+
+// TestRegisterWhenClosing verifies that CloseAll rejects further Register.
+func TestRegisterWhenClosing(t *testing.T) {
+ h := NewHub(5)
+ h.CloseAll()
+ conn := newTestConn(t)
+ _, err := h.Register("user-x", conn)
+ if err != ErrHubClosing {
+ t.Errorf("Register after CloseAll err = %v, want ErrHubClosing", err)
+ }
+ if !h.IsClosing() {
+ t.Error("IsClosing = false, want true")
+ }
+}
+
+// TestSendToUser verifies message delivery to a single user.
+func TestSendToUser(t *testing.T) {
+ h := NewHub(5)
+ conn := newTestConn(t)
+ c, err := h.Register("user-send", conn)
+ if err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ defer h.Unregister(c)
+
+ msg := []byte(`{"type":"message","event":"test"}`)
+ delivered := h.SendToUser("user-send", msg)
+ if delivered != 1 {
+ t.Errorf("SendToUser delivered = %d, want 1", delivered)
+ }
+
+ // Unknown user: 0 delivered.
+ delivered = h.SendToUser("user-absent", msg)
+ if delivered != 0 {
+ t.Errorf("SendToUser absent delivered = %d, want 0", delivered)
+ }
+}
+
+// TestBroadcast verifies message delivery to all connections.
+func TestBroadcast(t *testing.T) {
+ h := NewHub(5)
+ conn1 := newTestConn(t)
+ conn2 := newTestConn(t)
+ c1, _ := h.Register("u1", conn1)
+ c2, _ := h.Register("u2", conn2)
+ defer h.Unregister(c1)
+ defer h.Unregister(c2)
+
+ msg := []byte(`{"type":"message","event":"broadcast"}`)
+ reached := h.Broadcast(msg)
+ if reached != 2 {
+ t.Errorf("Broadcast reached = %d, want 2", reached)
+ }
+}
+
+// TestPresenceHooks verifies that Register/Unregister call the hooks.
+func TestPresenceHooks(t *testing.T) {
+ h := NewHub(5)
+ var mu sync.Mutex
+ registered := []string{}
+ unregistered := []string{}
+ h.SetPresenceHooks(
+ func(userID, connID string) {
+ mu.Lock()
+ defer mu.Unlock()
+ registered = append(registered, userID)
+ },
+ func(userID, connID string) {
+ mu.Lock()
+ defer mu.Unlock()
+ unregistered = append(unregistered, userID)
+ },
+ )
+
+ conn := newTestConn(t)
+ c, _ := h.Register("hook-user", conn)
+ h.Unregister(c)
+
+ if len(registered) != 1 || registered[0] != "hook-user" {
+ t.Errorf("registered = %v, want [hook-user]", registered)
+ }
+ if len(unregistered) != 1 || unregistered[0] != "hook-user" {
+ t.Errorf("unregistered = %v, want [hook-user]", unregistered)
+ }
+}
+
+// TestForEachUser verifies iteration over online users (used by ISSUE-058 rebuild).
+func TestForEachUser(t *testing.T) {
+ h := NewHub(5)
+ conn1 := newTestConn(t)
+ conn2 := newTestConn(t)
+ c1, _ := h.Register("u-a", conn1)
+ c2, _ := h.Register("u-b", conn2)
+ defer h.Unregister(c1)
+ defer h.Unregister(c2)
+
+ seen := map[string]bool{}
+ h.ForEachUser(func(userID string) {
+ seen[userID] = true
+ })
+ if len(seen) != 2 || !seen["u-a"] || !seen["u-b"] {
+ t.Errorf("ForEachUser seen = %v, want {u-a, u-b}", seen)
+ }
+}
+
+// TestConnectionSendOnClosed verifies Send returns false after Close.
+func TestConnectionSendOnClosed(t *testing.T) {
+ h := NewHub(5)
+ conn := newTestConn(t)
+ c, _ := h.Register("u-close", conn)
+ c.Close()
+ if ok := c.Send([]byte("x")); ok {
+ t.Error("Send after Close = true, want false")
+ }
+}
+
+// TestConnectionSendChannelFull verifies Send returns false when channel is full.
+func TestConnectionSendChannelFull(t *testing.T) {
+ h := NewHub(5)
+ conn := newTestConn(t)
+ c, _ := h.Register("u-full", conn)
+ defer h.Unregister(c)
+ // Fill the send channel (cap 64).
+ for i := 0; i < sendBufferSize; i++ {
+ if !c.Send([]byte("x")) {
+ t.Fatalf("Send #%d returned false unexpectedly", i)
+ }
+ }
+ // Next Send should drop.
+ if ok := c.Send([]byte("overflow")); ok {
+ t.Error("Send on full channel = true, want false")
+ }
+}
+
+// TestNewHubDefaultMaxConns verifies NewHub applies a default when maxConns <= 0.
+func TestNewHubDefaultMaxConns(t *testing.T) {
+ h := NewHub(0)
+ if h.maxConnsPerUser != 5 {
+ t.Errorf("NewHub(0) maxConnsPerUser = %d, want default 5", h.maxConnsPerUser)
+ }
+ h2 := NewHub(-1)
+ if h2.maxConnsPerUser != 5 {
+ t.Errorf("NewHub(-1) maxConnsPerUser = %d, want default 5", h2.maxConnsPerUser)
+ }
+}
diff --git a/services/push-gateway/internal/kafkaconsumer/consumer.go b/services/push-gateway/internal/kafkaconsumer/consumer.go
index fdc835e..006b778 100644
--- a/services/push-gateway/internal/kafkaconsumer/consumer.go
+++ b/services/push-gateway/internal/kafkaconsumer/consumer.go
@@ -120,6 +120,39 @@ func (c *Consumer) Close() error {
return c.reader.Close()
}
+// HealthCheck verifies that the Kafka broker is reachable. Used by /readyz as
+// a soft-failure probe: a failing check marks the consumer as degraded but
+// does not return 503 (per ARB-015 §17.4 / ISSUE-058).
+//
+// The check issues a metadata request for the consumer topic; on success the
+// broker is considered reachable. This is cheaper than a Lag() call which
+// requires partition assignment to have completed.
+func (c *Consumer) HealthCheck(ctx context.Context) error {
+ conn, err := kafka.DialContext(ctx, "tcp", c.reader.Config().Brokers[0])
+ if err != nil {
+ return fmt.Errorf("kafka: dial broker: %w", err)
+ }
+ defer conn.Close()
+ partitions, err := conn.ReadPartitions(c.topic)
+ if err != nil {
+ return fmt.Errorf("kafka: read partitions: %w", err)
+ }
+ if len(partitions) == 0 {
+ return fmt.Errorf("kafka: topic %q has no partitions", c.topic)
+ }
+ return nil
+}
+
+// Topic returns the configured topic name. Exposed for diagnostics.
+func (c *Consumer) Topic() string {
+ return c.topic
+}
+
+// GroupID returns the configured consumer group name.
+func (c *Consumer) GroupID() string {
+ return c.groupID
+}
+
// processMessage handles a single Kafka message with retry and idempotency.
func (c *Consumer) processMessage(ctx context.Context, msg kafka.Message) {
var event NotificationRequested
diff --git a/services/push-gateway/internal/ws/handler.go b/services/push-gateway/internal/ws/handler.go
index 1ffdbc8..d380a73 100644
--- a/services/push-gateway/internal/ws/handler.go
+++ b/services/push-gateway/internal/ws/handler.go
@@ -1,18 +1,79 @@
+// Package ws implements the WebSocket upgrade endpoint, the /internal/push
+// and /internal/broadcast HTTP APIs consumed by msg, and the
+// /internal/online/ presence query.
+//
+// Authentication:
+// - WebSocket /ws: JWT RS256 validated via shared-go/jwks (caching JWKS from
+// iam /.well-known/jwks.json, refreshed every 5 minutes). DevMode accepts
+// the literal "dev-token" returning a synthetic dev-user subject.
+// - /internal/*: X-Internal-Token header matched against INTERNAL_API_TOKEN
+// (ARB-015 §17.3 / president §7.2). DevMode skips the check.
+//
+// Heartbeat (RFC 6455 control frames, not text messages):
+// - Client sends Ping every 30s; gorilla/websocket auto-replies with Pong.
+// - Server sets a 60s read deadline; if no frame arrives the connection is
+// closed (idle timeout).
+// - Each Ping refreshes the Redis presence TTL via RefreshPresence.
+//
+// Per-user connection cap: enforced by Hub.Register (default 5). Exceeding it
+// returns a 429 PUSH_TOO_MANY_CONNECTIONS for HTTP callers and a close frame
+// (1008 policy violation) for WebSocket callers.
package ws
import (
+ "context"
"encoding/json"
"errors"
- "log"
+ "fmt"
"net/http"
"strings"
+ "time"
+ "github.com/edu-cloud/push-gateway/internal/config"
"github.com/edu-cloud/push-gateway/internal/hub"
+ "github.com/edu-cloud/push-gateway/internal/observability"
+ "github.com/edu-cloud/push-gateway/internal/redisclient"
+ "github.com/edu-cloud/shared-go/jwks"
"github.com/gin-gonic/gin"
"github.com/golang-jwt/jwt/v5"
"github.com/gorilla/websocket"
)
+// internalTokenHeader is the canonical header for /internal/* authentication
+// (ARB-015 §17.3 / president §7.2). Was X-Internal-Key in pre-P5 builds.
+const internalTokenHeader = "X-Internal-Token"
+
+// readBufferSize / writeBufferSize tune gorilla/websocket's internal buffers.
+const (
+ readBufferSize = 4096
+ writeBufferSize = 4096
+)
+
+// idleTimeout is the maximum interval with no frames before the connection is
+// considered dead. Should be at least 2x the client's heartbeat interval.
+const idleTimeout = 60 * time.Second
+
+// writeTimeout bounds blocking writes from the writer goroutine.
+const writeTimeout = 10 * time.Second
+
+// closeHandshakeTimeout bounds the close-frame exchange before forcing close.
+const closeHandshakeTimeout = 5 * time.Second
+
+// pushResponse is the ActionState-shaped envelope returned by /internal/push
+// and /internal/broadcast (see 02 §4.2).
+type pushResponse struct {
+ Success bool `json:"success"`
+ Delivered bool `json:"delivered,omitempty"`
+ Online bool `json:"online,omitempty"`
+ Reached int `json:"reached,omitempty"`
+ Error *errBody `json:"error,omitempty"`
+}
+
+type errBody struct {
+ Code string `json:"code"`
+ Message string `json:"message"`
+}
+
var (
errMissingToken = errors.New("missing token")
errInvalidToken = errors.New("invalid token")
@@ -20,148 +81,349 @@ var (
errMissingUserID = errors.New("missing user id")
)
-var upgrader = websocket.Upgrader{
- CheckOrigin: func(r *http.Request) bool {
- return true // P5 骨架，生产环境需校验 origin
- },
-}
-
-// internalAPIKeyHeader 是内部 API 鉴权请求头名称
-const internalAPIKeyHeader = "X-Internal-Key"
-
-// Handler 处理 WebSocket 升级与内部推送 API
+// Handler owns the WebSocket upgrade path and the internal HTTP APIs.
type Handler struct {
hub *hub.Hub
- jwtSecret string
- devMode bool
- internalAPIKey string
+ cfg *config.Config
+ redis *redisclient.Client
+ jwksFetcher *jwks.Fetcher
+ metrics *observability.Metrics
+ upgrader websocket.Upgrader
}
-// NewHandler 创建 Handler 实例
-func NewHandler(h *hub.Hub, jwtSecret string, devMode bool, internalAPIKey string) *Handler {
- return &Handler{hub: h, jwtSecret: jwtSecret, devMode: devMode, internalAPIKey: internalAPIKey}
+// NewHandler constructs a Handler. cfg supplies auth tokens and origin
+// whitelist; redis may be nil in DevMode (presence and cross-instance fanout
+// are skipped). jwksFetcher may be nil in DevMode (HS256 fallback).
+func NewHandler(h *hub.Hub, cfg *config.Config, r *redisclient.Client, m *observability.Metrics) *Handler {
+ allowed := make(map[string]struct{}, len(cfg.AllowedOrigins))
+ for _, o := range cfg.AllowedOrigins {
+ allowed[o] = struct{}{}
+ }
+ var fetcher *jwks.Fetcher
+ if cfg.JWKSURL != "" {
+ fetcher = jwks.NewFetcher(cfg.JWKSURL)
+ }
+ return &Handler{
+ hub: h,
+ cfg: cfg,
+ redis: r,
+ jwksFetcher: fetcher,
+ metrics: m,
+ upgrader: websocket.Upgrader{
+ ReadBufferSize: readBufferSize,
+ WriteBufferSize: writeBufferSize,
+ CheckOrigin: buildOriginChecker(allowed, cfg.DevMode),
+ },
+ }
}
-// HandleWebSocket 升级 HTTP 为 WebSocket，保持长连接并处理心跳
+// buildOriginChecker returns a CheckOrigin function that allows only the
+// configured whitelist. In DevMode with an empty whitelist all origins are
+// accepted (local development convenience).
+func buildOriginChecker(allowed map[string]struct{}, devMode bool) func(r *http.Request) bool {
+ return func(r *http.Request) bool {
+ origin := r.Header.Get("Origin")
+ if origin == "" {
+ // Non-browser clients (curl, Postman) have no Origin header.
+ return true
+ }
+ if devMode && len(allowed) == 0 {
+ return true
+ }
+ _, ok := allowed[origin]
+ return ok
+ }
+}
+
+// HandleWebSocket upgrades an HTTP request to a WebSocket connection. The
+// caller must provide a valid JWT (RS256 in production, dev-token in DevMode)
+// via the ?token= query parameter or the Authorization: Bearer header.
func (h *Handler) HandleWebSocket(c *gin.Context) {
userID, err := h.authenticate(c)
if err != nil {
- c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
+ c.AbortWithStatusJSON(http.StatusUnauthorized, pushResponse{
+ Success: false,
+ Error: &errBody{Code: "PUSH_UNAUTHORIZED", Message: err.Error()},
+ })
return
}
- conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
+ conn, err := h.upgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
- log.Printf("[ws] upgrade failed, user=%s: %v", userID, err)
+ // Upgrade already wrote the error response via c.Writer.
+ observability.Logger().Warn("ws upgrade failed",
+ "user_id", userID, "err", err)
return
}
- defer conn.Close()
+ // Apply per-connection limits.
+ conn.SetReadLimit(64 * 1024) // 64KB max message size (02 §11)
- connPtr := h.hub.Register(userID, conn)
+ connPtr, err := h.hub.Register(userID, conn)
+ if err != nil {
+ closeCode := websocket.ClosePolicyViolation
+ if errors.Is(err, hub.ErrHubClosing) {
+ closeCode = websocket.CloseGoingAway
+ }
+ _ = conn.WriteControl(
+ websocket.CloseMessage,
+ websocket.FormatCloseMessage(closeCode, err.Error()),
+ time.Now().Add(closeHandshakeTimeout),
+ )
+ _ = conn.Close()
+ if errors.Is(err, hub.ErrTooManyConnections) {
+ h.metrics.IncDisconnect("too_many_connections")
+ }
+ return
+ }
defer h.hub.Unregister(connPtr)
- // 写协程：从 send chan 读取并写入 WebSocket，避免并发写
- go func() {
- for msg := range connPtr.Outgoing() {
- if err := conn.WriteMessage(websocket.TextMessage, msg); err != nil {
- log.Printf("[ws] write failed, user=%s: %v", userID, err)
+ // Writer goroutine: consumes connPtr.Outgoing() and writes to the socket.
+ // Single writer to satisfy gorilla/websocket's "no concurrent writers" rule.
+ go h.writerLoop(connPtr)
+
+ // Reader loop: handles Ping/Pong control frames and idle timeout.
+ h.readerLoop(connPtr)
+}
+
+// writerLoop drains the connection's send channel and writes each message as a
+// TextMessage. Exits when the channel is closed by Hub.Unregister / CloseAll.
+func (h *Handler) writerLoop(c *hub.Connection) {
+ conn := c.Conn()
+ for msg := range c.Outgoing() {
+ _ = conn.SetWriteDeadline(time.Now().Add(writeTimeout))
+ if err := conn.WriteMessage(websocket.TextMessage, msg); err != nil {
+ observability.Logger().Debug("ws write failed",
+ "user_id", c.UserID, "conn_id", c.ConnID, "err", err)
+ return
+ }
+ }
+}
+
+// readerLoop reads frames until the connection closes. It enforces the 60s
+// idle timeout via SetReadDeadline and refreshes the Redis presence TTL on
+// each Pong handler invocation (RFC 6455 control frame heartbeat).
+func (h *Handler) readerLoop(c *hub.Connection) {
+ conn := c.Conn()
+ _ = conn.SetReadDeadline(time.Now().Add(idleTimeout))
+ conn.SetPongHandler(func(appData string) error {
+ _ = conn.SetReadDeadline(time.Now().Add(idleTimeout))
+ c.TouchPing()
+ h.metrics.Heartbeats.Inc()
+ if h.redis != nil {
+ ctx, cancel := contextWithTimeout(500 * time.Millisecond)
+ defer cancel()
+ _ = h.redis.RefreshPresence(ctx, c.UserID)
+ }
+ return nil
+ })
+
+ for {
+ messageType, _, err := conn.ReadMessage()
+ if err != nil {
+ if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
+ h.metrics.IncDisconnect("closed")
+ } else if websocket.IsUnexpectedCloseError(err) {
+ h.metrics.IncDisconnect("error")
+ observability.Logger().Debug("ws read failed",
+ "user_id", c.UserID, "conn_id", c.ConnID, "err", err)
+ } else {
+ h.metrics.IncDisconnect("idle")
+ }
+ return
+ }
+ // We accept only Ping/Pong control frames (handled above) and ignore
+ // any client-sent Text/Binary messages. Clients must NOT send text
+ // "ping" messages (legacy protocol removed in P5).
+ if messageType == websocket.TextMessage || messageType == websocket.BinaryMessage {
+ // Acknowledge application-level ack messages with a no-op; ignore
+ // everything else. Future P6 reconnect protocol may use this path.
+ continue
+ }
+ }
+}
+
+// PushHandler implements POST /internal/push: directed push to a single user.
+// Response carries delivered + online so msg (ai10) can decide offline fallback.
+func (h *Handler) PushHandler(c *gin.Context) {
+ if !h.checkInternalToken(c) {
+ return
+ }
+
+ var req struct {
+ UserID string `json:"user_id" binding:"required"`
+ Event string `json:"event" binding:"required"`
+ Data map[string]any `json:"data"`
+ TTL *int `json:"ttl,omitempty"`
+ }
+ if err := c.ShouldBindJSON(&req); err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, pushResponse{
+ Success: false,
+ Error: &errBody{Code: "PUSH_INVALID_REQUEST", Message: err.Error()},
+ })
+ return
+ }
+
+ message, err := buildMessage(req.Event, req.Data)
+ if err != nil {
+ c.AbortWithStatusJSON(http.StatusBadRequest, pushResponse{
+ Success: false,
+ Error: &errBody{Code: "PUSH_INVALID_PAYLOAD", Message: err.Error()},
+ })
+ return
+ }
+
+ // Try local delivery first.
+ delivered := h.hub.SendToUser(req.UserID, message)
+ if delivered > 0 {
+ h.metrics.IncPushed(req.Event, "delivered")
+ c.JSON(http.StatusOK, pushResponse{Success: true, Delivered: true, Online: true})
+ return
+ }
+
+ // Fall back to cross-instance Redis Pub/Sub fanout.
+ if h.redis != nil {
+ ctx, cancel := contextWithTimeout(2 * time.Second)
+ defer cancel()
+ online, _, err := h.redis.IsOnline(ctx, req.UserID)
+ if err != nil {
+ observability.Logger().Warn("redis IsOnline failed",
+ "user_id", req.UserID, "err", err)
+ // Treat Redis errors as "online unknown" — try publishing anyway.
+ online = true
+ }
+ if online {
+ err := h.redis.PublishUser(ctx, req.UserID, redisclient.CrossInstanceMessage{
+ Event: req.Event,
+ Data: mustMarshal(req.Data),
+ })
+ if err != nil {
+ observability.Logger().Warn("redis PublishUser failed",
+ "user_id", req.UserID, "err", err)
+ c.JSON(http.StatusOK, pushResponse{Success: false, Delivered: false, Online: true})
return
}
+ // Pub/Sub is fire-and-forget: assume delivered if any instance held the user.
+ h.metrics.IncPushed(req.Event, "delivered")
+ c.JSON(http.StatusOK, pushResponse{Success: true, Delivered: true, Online: true})
+ return
}
- }()
-
- // 读取循环（保持连接，处理心跳）
- for {
- _, msg, err := conn.ReadMessage()
- if err != nil {
- if !websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
- log.Printf("[ws] read failed, user=%s: %v", userID, err)
- }
- break
- }
- if strings.ToLower(string(msg)) == "ping" {
- connPtr.Send([]byte("pong"))
- }
+ // Not online anywhere — msg should fall back to offline push (SMS/email).
+ h.metrics.IncPushed(req.Event, "offline")
+ c.JSON(http.StatusOK, pushResponse{Success: true, Delivered: false, Online: false})
+ return
}
+
+ // No Redis available (DevMode): report offline so msg falls back.
+ h.metrics.IncPushed(req.Event, "offline")
+ c.JSON(http.StatusOK, pushResponse{Success: true, Delivered: false, Online: false})
}
-// PushHandler 接收来自 Msg 服务的定向推送请求
-func (h *Handler) PushHandler(c *gin.Context) {
- if !h.checkInternalAPIKey(c) {
- c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{
- "success": false,
- "error": gin.H{"code": "UNAUTHORIZED", "message": "invalid or missing internal api key"},
- })
- return
- }
-
- var req struct {
- UserID string `json:"userId"`
- Event string `json:"event"`
- Data map[string]any `json:"data"`
- }
- if err := c.ShouldBindJSON(&req); err != nil {
- c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": gin.H{"code": "INVALID_REQUEST", "message": err.Error()}})
- return
- }
-
- message, err := buildMessage(req.Event, req.Data)
- if err != nil {
- c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": gin.H{"code": "INVALID_PAYLOAD", "message": err.Error()}})
- return
- }
-
- if err := h.hub.SendToUser(req.UserID, message); err != nil {
- c.JSON(http.StatusInternalServerError, gin.H{"success": false, "error": gin.H{"code": "PUSH_FAILED", "message": err.Error()}})
- return
- }
-
- c.JSON(http.StatusOK, gin.H{"success": true})
-}
-
-// BroadcastHandler 接收来自 Msg 服务的广播请求
+// BroadcastHandler implements POST /internal/broadcast: push to every online
+// client across all instances.
func (h *Handler) BroadcastHandler(c *gin.Context) {
- if !h.checkInternalAPIKey(c) {
- c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{
- "success": false,
- "error": gin.H{"code": "UNAUTHORIZED", "message": "invalid or missing internal api key"},
- })
+ if !h.checkInternalToken(c) {
return
}
var req struct {
- Event string `json:"event"`
- Data map[string]any `json:"data"`
+ Event string `json:"event" binding:"required"`
+ Data map[string]any `json:"data"`
+ Filter map[string]any `json:"filter,omitempty"`
}
if err := c.ShouldBindJSON(&req); err != nil {
- c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": gin.H{"code": "INVALID_REQUEST", "message": err.Error()}})
+ c.AbortWithStatusJSON(http.StatusBadRequest, pushResponse{
+ Success: false,
+ Error: &errBody{Code: "PUSH_INVALID_REQUEST", Message: err.Error()},
+ })
return
}
message, err := buildMessage(req.Event, req.Data)
if err != nil {
- c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": gin.H{"code": "INVALID_PAYLOAD", "message": err.Error()}})
+ c.AbortWithStatusJSON(http.StatusBadRequest, pushResponse{
+ Success: false,
+ Error: &errBody{Code: "PUSH_INVALID_PAYLOAD", Message: err.Error()},
+ })
return
}
- h.hub.Broadcast(message)
- c.JSON(http.StatusOK, gin.H{"success": true})
+ local := h.hub.Broadcast(message)
+
+ if h.redis != nil {
+ ctx, cancel := contextWithTimeout(2 * time.Second)
+ defer cancel()
+ _ = h.redis.PublishBroadcast(ctx, redisclient.CrossInstanceMessage{
+ Event: req.Event,
+ Data: mustMarshal(req.Data),
+ })
+ }
+
+ h.metrics.IncPushed(req.Event, "delivered")
+ c.JSON(http.StatusOK, pushResponse{Success: true, Reached: local})
}
-// checkInternalAPIKey 校验内部 API 请求头 X-Internal-Key。
-// DevMode 下跳过校验；非 DevMode 下要求 INTERNAL_API_KEY 已配置且与请求头匹配。
-func (h *Handler) checkInternalAPIKey(c *gin.Context) bool {
- if h.devMode {
+// OnlineHandler implements GET /internal/online/: returns whether the
+// user has any live connection across the cluster.
+func (h *Handler) OnlineHandler(c *gin.Context) {
+ if !h.checkInternalToken(c) {
+ return
+ }
+ userID := c.Param("userID")
+ if userID == "" {
+ c.AbortWithStatusJSON(http.StatusBadRequest, pushResponse{
+ Success: false,
+ Error: &errBody{Code: "PUSH_INVALID_REQUEST", Message: "missing userID"},
+ })
+ return
+ }
+
+ // Local shortcut.
+ if h.hub.HasUser(userID) {
+ c.JSON(http.StatusOK, gin.H{"online": true, "instances": []string{h.cfg.InstanceID}})
+ return
+ }
+
+ if h.redis != nil {
+ ctx, cancel := contextWithTimeout(1 * time.Second)
+ defer cancel()
+ online, members, err := h.redis.IsOnline(ctx, userID)
+ if err != nil {
+ observability.Logger().Warn("redis IsOnline failed",
+ "user_id", userID, "err", err)
+ c.JSON(http.StatusOK, gin.H{"online": false, "instances": []string{}})
+ return
+ }
+ c.JSON(http.StatusOK, gin.H{"online": online, "instances": members})
+ return
+ }
+ c.JSON(http.StatusOK, gin.H{"online": false, "instances": []string{}})
+}
+
+// checkInternalToken validates the X-Internal-Token header. Returns false when
+// the request has been aborted (caller should return immediately).
+func (h *Handler) checkInternalToken(c *gin.Context) bool {
+ if h.cfg.DevMode {
return true
}
- if h.internalAPIKey == "" {
- log.Println("[ws] internal api key not configured, rejecting request")
+ if h.cfg.InternalAPIToken == "" {
+ c.AbortWithStatusJSON(http.StatusUnauthorized, pushResponse{
+ Success: false,
+ Error: &errBody{Code: "PUSH_UNAUTHORIZED", Message: "internal token not configured"},
+ })
return false
}
- return c.GetHeader(internalAPIKeyHeader) == h.internalAPIKey
+ if c.GetHeader(internalTokenHeader) != h.cfg.InternalAPIToken {
+ c.AbortWithStatusJSON(http.StatusUnauthorized, pushResponse{
+ Success: false,
+ Error: &errBody{Code: "PUSH_UNAUTHORIZED", Message: "invalid or missing internal token"},
+ })
+ return false
+ }
+ return true
}
-// authenticate 校验 WebSocket 连接的 JWT 或 dev token
+// authenticate validates the WebSocket JWT. Production uses RS256 via the
+// shared-go/jwks Fetcher; DevMode accepts the literal "dev-token" string.
func (h *Handler) authenticate(c *gin.Context) (string, error) {
tokenStr := c.Query("token")
if tokenStr == "" {
@@ -173,37 +435,81 @@ func (h *Handler) authenticate(c *gin.Context) (string, error) {
return "", errMissingToken
}
- // DEV_MODE 下接受 dev-token，便于本地联调
- if h.devMode && tokenStr == "dev-token" {
+ // DevMode shortcut: dev-token maps to a synthetic local user.
+ if h.cfg.DevMode && tokenStr == "dev-token" {
return "dev-user", nil
}
- token, err := jwt.Parse(tokenStr, func(t *jwt.Token) (any, error) {
- if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
- return nil, jwt.ErrSignatureInvalid
+ // RS256 via shared-go/jwks (preferred, requires iam /.well-known/jwks.json).
+ if h.jwksFetcher != nil {
+ claims, err := h.jwksFetcher.ValidateToken(tokenStr)
+ if err != nil {
+ return "", errInvalidToken
}
- return []byte(h.jwtSecret), nil
- })
- if err != nil || !token.Valid {
- return "", errInvalidToken
+ if claims.UserID == "" {
+ return "", errMissingUserID
+ }
+ return claims.UserID, nil
}
- claims, ok := token.Claims.(jwt.MapClaims)
- if !ok {
- return "", errInvalidClaims
+ // DevMode without JWKS: fall back to HS256 with the configured secret so
+ // local integration tests can still sign tokens.
+ if h.cfg.DevMode && h.cfg.JWTSecret != "" {
+ return validateHS256(tokenStr, h.cfg.JWTSecret)
}
-
- userID, ok := claims["sub"].(string)
- if !ok {
- return "", errMissingUserID
- }
- return userID, nil
+ return "", errInvalidToken
}
-// buildMessage 构造推送 JSON 消息体
+// buildMessage constructs the application-layer WebSocket message envelope
+// (02 §4.1): {type:"message", event, data, timestamp}.
func buildMessage(event string, data map[string]any) ([]byte, error) {
return json.Marshal(map[string]any{
- "event": event,
- "data": data,
+ "type": "message",
+ "event": event,
+ "data": data,
+ "timestamp": time.Now().UTC().Format(time.RFC3339),
})
}
+
+// mustMarshal is a best-effort JSON encoder for the Pub/Sub payload; on error
+// it returns null, which the subscriber handles as empty data.
+func mustMarshal(data map[string]any) []byte {
+ if data == nil {
+ return []byte("null")
+ }
+ b, err := json.Marshal(data)
+ if err != nil {
+ return []byte("null")
+ }
+ return b
+}
+
+// contextWithTimeout returns a fresh context + cancel pair with the given
+// timeout. Used by HTTP handlers and the reader loop for short-lived Redis
+// round-trips (presence refresh, IsOnline, Publish).
+func contextWithTimeout(timeout time.Duration) (context.Context, context.CancelFunc) {
+ return context.WithTimeout(context.Background(), timeout)
+}
+
+// validateHS256 parses and validates an HS256 JWT signed with secret. Used
+// only in DevMode when no JWKS endpoint is configured (local integration
+// tests). Returns the user_id claim on success.
+func validateHS256(tokenStr, secret string) (string, error) {
+ claims := &jwks.Claims{}
+ parsed, err := jwt.ParseWithClaims(tokenStr, claims, func(t *jwt.Token) (any, error) {
+ if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
+ return nil, fmt.Errorf("ws: unexpected signing method %v", t.Header["alg"])
+ }
+ return []byte(secret), nil
+ })
+ if err != nil {
+ return "", errInvalidToken
+ }
+ if !parsed.Valid {
+ return "", errInvalidToken
+ }
+ if claims.UserID == "" {
+ return "", errMissingUserID
+ }
+ return claims.UserID, nil
+}
diff --git a/services/push-gateway/internal/ws/handler_test.go b/services/push-gateway/internal/ws/handler_test.go
new file mode 100644
index 0000000..e76a4f0
--- /dev/null
+++ b/services/push-gateway/internal/ws/handler_test.go
@@ -0,0 +1,196 @@
+package ws
+
+import (
+ "encoding/json"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/edu-cloud/shared-go/jwks"
+ "github.com/golang-jwt/jwt/v5"
+)
+
+// TestBuildMessage verifies the WebSocket message envelope format (02 §4.1).
+func TestBuildMessage(t *testing.T) {
+ data := map[string]any{"title": "hello", "body": "world"}
+ msg, err := buildMessage("notification.created", data)
+ if err != nil {
+ t.Fatalf("buildMessage: %v", err)
+ }
+
+ var payload map[string]any
+ if err := json.Unmarshal(msg, &payload); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if payload["type"] != "message" {
+ t.Errorf("type = %v, want message", payload["type"])
+ }
+ if payload["event"] != "notification.created" {
+ t.Errorf("event = %v, want notification.created", payload["event"])
+ }
+ if payload["timestamp"] == nil {
+ t.Error("timestamp missing")
+ }
+ dataField, ok := payload["data"].(map[string]any)
+ if !ok {
+ t.Fatalf("data should be a map, got %T", payload["data"])
+ }
+ if dataField["title"] != "hello" {
+ t.Errorf("data.title = %v, want hello", dataField["title"])
+ }
+}
+
+// TestBuildMessageNilData verifies nil data is handled gracefully.
+func TestBuildMessageNilData(t *testing.T) {
+ msg, err := buildMessage("test.event", nil)
+ if err != nil {
+ t.Fatalf("buildMessage: %v", err)
+ }
+ var payload map[string]any
+ if err := json.Unmarshal(msg, &payload); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if payload["data"] != nil {
+ t.Errorf("data = %v, want nil", payload["data"])
+ }
+}
+
+// TestMustMarshal verifies best-effort JSON encoding.
+func TestMustMarshal(t *testing.T) {
+ // nil -> "null"
+ if got := mustMarshal(nil); string(got) != "null" {
+ t.Errorf("mustMarshal(nil) = %q, want null", got)
+ }
+ // valid map -> JSON
+ got := mustMarshal(map[string]any{"a": 1})
+ var m map[string]any
+ if err := json.Unmarshal(got, &m); err != nil {
+ t.Fatalf("mustMarshal result unmarshal: %v", err)
+ }
+ if m["a"].(float64) != 1 {
+ t.Errorf("mustMarshal.a = %v, want 1", m["a"])
+ }
+}
+
+// TestBuildOriginChecker verifies origin whitelist logic.
+func TestBuildOriginChecker(t *testing.T) {
+ allowed := map[string]struct{}{
+ "http://localhost:3000": {},
+ "https://app.example.com": {},
+ }
+
+ // Non-devMode: strict whitelist.
+ checker := buildOriginChecker(allowed, false)
+
+ tests := []struct {
+ name string
+ origin string
+ want bool
+ }{
+ {"empty origin (non-browser)", "", true},
+ {"allowed origin", "http://localhost:3000", true},
+ {"another allowed", "https://app.example.com", true},
+ {"disallowed origin", "http://evil.com", false},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ req := httptest.NewRequest("GET", "/ws", nil)
+ if tc.origin != "" {
+ req.Header.Set("Origin", tc.origin)
+ }
+ if got := checker(req); got != tc.want {
+ t.Errorf("checker(origin=%q) = %v, want %v", tc.origin, got, tc.want)
+ }
+ })
+ }
+
+ // DevMode with empty whitelist: allow all.
+ devChecker := buildOriginChecker(map[string]struct{}{}, true)
+ req := httptest.NewRequest("GET", "/ws", nil)
+ req.Header.Set("Origin", "http://anything.com")
+ if !devChecker(req) {
+ t.Error("devMode empty whitelist should allow all origins")
+ }
+}
+
+// TestValidateHS256 verifies HS256 JWT validation in DevMode fallback.
+func TestValidateHS256(t *testing.T) {
+ secret := "test-secret"
+
+ // Valid token with user_id claim.
+ claims := jwks.Claims{
+ UserID: "user-123",
+ RegisteredClaims: jwt.RegisteredClaims{
+ ExpiresAt: jwt.NewNumericDate(time.Now().Add(1 * time.Hour)),
+ },
+ }
+ token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+ tokenStr, err := token.SignedString([]byte(secret))
+ if err != nil {
+ t.Fatalf("SignedString: %v", err)
+ }
+
+ userID, err := validateHS256(tokenStr, secret)
+ if err != nil {
+ t.Fatalf("validateHS256: %v", err)
+ }
+ if userID != "user-123" {
+ t.Errorf("userID = %q, want user-123", userID)
+ }
+
+ // Wrong secret -> error.
+ _, err = validateHS256(tokenStr, "wrong-secret")
+ if err == nil {
+ t.Error("validateHS256 with wrong secret should fail")
+ }
+
+ // Expired token -> error.
+ expiredClaims := jwks.Claims{
+ UserID: "user-exp",
+ RegisteredClaims: jwt.RegisteredClaims{
+ ExpiresAt: jwt.NewNumericDate(time.Now().Add(-1 * time.Hour)),
+ },
+ }
+ expiredToken := jwt.NewWithClaims(jwt.SigningMethodHS256, expiredClaims)
+ expiredStr, _ := expiredToken.SignedString([]byte(secret))
+ _, err = validateHS256(expiredStr, secret)
+ if err == nil {
+ t.Error("validateHS256 with expired token should fail")
+ }
+
+ // Missing user_id -> error.
+ noUserClaims := jwks.Claims{
+ RegisteredClaims: jwt.RegisteredClaims{
+ ExpiresAt: jwt.NewNumericDate(time.Now().Add(1 * time.Hour)),
+ },
+ }
+ noUserToken := jwt.NewWithClaims(jwt.SigningMethodHS256, noUserClaims)
+ noUserStr, _ := noUserToken.SignedString([]byte(secret))
+ _, err = validateHS256(noUserStr, secret)
+ if err != errMissingUserID {
+ t.Errorf("validateHS256 without user_id err = %v, want errMissingUserID", err)
+ }
+}
+
+// TestContextWithTimeout verifies the helper returns a working context.
+func TestContextWithTimeout(t *testing.T) {
+ ctx, cancel := contextWithTimeout(100 * time.Millisecond)
+ defer cancel()
+ if ctx == nil {
+ t.Fatal("context is nil")
+ }
+ // Context should not be done immediately.
+ select {
+ case <-ctx.Done():
+ t.Fatal("context done immediately")
+ default:
+ }
+ // Wait for timeout.
+ time.Sleep(150 * time.Millisecond)
+ select {
+ case <-ctx.Done():
+ // expected
+ default:
+ t.Error("context not done after timeout")
+ }
+}
diff --git a/services/push-gateway/main.go b/services/push-gateway/main.go
index 7dc2760..07f2993 100644
--- a/services/push-gateway/main.go
+++ b/services/push-gateway/main.go
@@ -1,8 +1,33 @@
+// Command push-gateway is the Edu platform's real-time WebSocket push gateway.
+//
+// It maintains per-user WebSocket connections, delivers directed and broadcast
+// messages from msg (via HTTP /internal/* APIs and Kafka), and synchronizes
+// online presence across instances via Redis Pub/Sub.
+//
+// Startup sequence:
+// 1. config.Load (env vars, DevMode detection)
+// 2. observability.InitLogger (slog JSON/text)
+// 3. observability.InitTracer (shared-go/tracer → OTLP)
+// 4. observability.NewMetrics (Prometheus)
+// 5. hub.NewHub (in-memory connection registry)
+// 6. redisclient.New + SetHub + RebuildPresenceOnStartup (ISSUE-058)
+// 7. redisclient.SubscribeAll (cross-instance Pub/Sub fanout)
+// 8. kafkaconsumer.New + goroutine Run (edu.notification.requested)
+// 9. ws.NewHandler (JWT RS256, X-Internal-Token, WebSocket upgrade)
+// 10. health.NewReadyzer (/readyz soft-failure probe)
+// 11. gin router + http.Server
+//
+// Graceful shutdown (SIGINT/SIGTERM):
+// 1. Hub.CloseAll (send close frame 1001 to every live connection)
+// 2. http.Server.Shutdown (stop accepting new HTTP/WebSocket requests)
+// 3. Kafka consumer goroutine cancel + reader.Close
+// 4. Redis Pub/Sub cancel + client.Close
+// 5. Tracer shutdown (flush pending spans)
package main
import (
"context"
- "log"
+ "errors"
"net/http"
"os"
"os/signal"
@@ -10,76 +35,195 @@ import (
"time"
"github.com/edu-cloud/push-gateway/internal/config"
+ "github.com/edu-cloud/push-gateway/internal/health"
"github.com/edu-cloud/push-gateway/internal/hub"
+ "github.com/edu-cloud/push-gateway/internal/kafkaconsumer"
"github.com/edu-cloud/push-gateway/internal/observability"
+ "github.com/edu-cloud/push-gateway/internal/redisclient"
"github.com/edu-cloud/push-gateway/internal/ws"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
)
+const serviceName = "push-gateway"
+
+// shutdownTimeout bounds the graceful shutdown of the HTTP server. After this
+// window in-flight requests are forcibly closed. WebSocket long-poll
+// connections are drained by Hub.CloseAll before this timer starts.
+const shutdownTimeout = 10 * time.Second
+
func main() {
cfg := config.Load()
- // 初始化 OpenTelemetry tracer（endpoint 为空时自动跳过）
- tracerShutdown := observability.InitTracer("push-gateway", cfg.OTLPEndpoint)
+
+ // 1. Structured logger (slog).
+ logger := observability.InitLogger(cfg.DevMode)
+ logger.Info("push-gateway starting",
+ "dev_mode", cfg.DevMode,
+ "instance_id", cfg.InstanceID,
+ "port", cfg.Port,
+ "max_conns_per_user", cfg.MaxConnsPerUser,
+ "jwks_url", cfg.JWKSURL,
+ "redis_url", cfg.RedisURL,
+ "kafka_brokers", cfg.KafkaBrokers,
+ "kafka_topic", cfg.KafkaNotificationTopic,
+ )
+
+ // 2. OpenTelemetry tracer (best-effort).
+ tracerShutdown := observability.InitTracer(serviceName, cfg.OTLPEndpoint)
defer tracerShutdown()
+ // 3. Prometheus metrics.
+ metrics := observability.NewMetrics()
+
+ // 4. Hub: in-memory connection registry.
+ h := hub.NewHub(cfg.MaxConnsPerUser)
+
+ // 5. Redis client (optional in DevMode). When present, wire presence
+ // hooks to the Hub and rebuild the online SET (ISSUE-058).
+ var redisClient *redisclient.Client
+ if cfg.RedisURL != "" {
+ rc, err := redisclient.New(cfg.RedisURL, cfg.InstanceID, metrics)
+ if err != nil {
+ logger.Error("redis connect failed; running without cross-instance fanout",
+ "err", err, "url", cfg.RedisURL)
+ } else {
+ redisClient = rc
+ redisClient.SetHub(h)
+ rebuildCtx, rebuildCancel := context.WithTimeout(context.Background(), 5*time.Second)
+ if err := redisClient.RebuildPresenceOnStartup(rebuildCtx); err != nil {
+ logger.Warn("redis presence rebuild failed (continuing)", "err", err)
+ }
+ rebuildCancel()
+ logger.Info("redis presence synced", "url", cfg.RedisURL)
+ }
+ } else {
+ logger.Warn("REDIS_URL empty; cross-instance fanout disabled")
+ }
+
+ // 6. Redis Pub/Sub subscriber (cross-instance message fanout).
+ var pubsubCancel func() error
+ if redisClient != nil {
+ subCtx, subCancel := context.WithCancel(context.Background())
+ defer subCancel()
+ var err error
+ pubsubCancel, err = redisClient.SubscribeAll(subCtx)
+ if err != nil {
+ logger.Error("redis SubscribeAll failed; cross-instance fanout disabled",
+ "err", err)
+ } else {
+ logger.Info("redis pubsub subscriber active")
+ }
+ }
+
+ // 7. Kafka consumer (edu.notification.requested). Started only when at
+ // least one broker is configured; otherwise the HTTP /internal/push API
+ // is the only delivery channel.
+ var kafkaConsumer *kafkaconsumer.Consumer
+ var kafkaCancel context.CancelFunc
+ if len(cfg.KafkaBrokers) > 0 && cfg.KafkaNotificationTopic != "" {
+ kafkaConsumer = kafkaconsumer.New(kafkaconsumer.Config{
+ Brokers: cfg.KafkaBrokers,
+ Topic: cfg.KafkaNotificationTopic,
+ GroupID: cfg.KafkaConsumerGroup,
+ }, h, redisClient, metrics)
+ kafkaCtx, cancel := context.WithCancel(context.Background())
+ kafkaCancel = cancel
+ go func() {
+ logger.Info("kafka consumer starting",
+ "topic", cfg.KafkaNotificationTopic, "group", cfg.KafkaConsumerGroup)
+ if err := kafkaConsumer.Run(kafkaCtx); err != nil &&
+ !errors.Is(err, context.Canceled) {
+ logger.Error("kafka consumer exited with error", "err", err)
+ }
+ }()
+ } else {
+ logger.Warn("KAFKA_BROKERS empty; notification consumption disabled")
+ }
+
+ // 8. WebSocket + internal HTTP handlers.
+ wsHandler := ws.NewHandler(h, cfg, redisClient, metrics)
+
+ // 9. /readyz probe (soft failure per ARB-015 §17.4).
+ readyzer := health.NewReadyzer(h, redisClient, kafkaConsumer, serviceName, cfg.InstanceID)
+
+ // 10. Gin router.
gin.SetMode(gin.ReleaseMode)
-
- h := hub.NewHub()
- wsHandler := ws.NewHandler(h, cfg.JWTSecret, cfg.DevMode, cfg.InternalAPIKey)
-
r := gin.New()
r.Use(gin.Recovery())
- // OpenTelemetry 自动埋点（HTTP 请求/响应 span）
- r.Use(otelgin.Middleware("push-gateway"))
+ r.Use(otelgin.Middleware(serviceName))
- r.GET("/healthz", func(c *gin.Context) {
- c.JSON(200, gin.H{"status": "ok", "service": "push-gateway"})
- })
-
- // 就绪探针：检查 WebSocket hub 状态
- r.GET("/readyz", func(c *gin.Context) {
- c.JSON(200, gin.H{
- "status": "ok",
- "service": "push-gateway",
- "connections": h.ClientCount(),
- })
- })
-
- // Prometheus 指标端点
+ // Liveness (no auth, no dependency checks).
+ r.GET("/healthz", health.Healthz(serviceName))
+ // Readiness (soft failure on Redis/Kafka).
+ r.GET("/readyz", readyzer.Handler)
+ // Prometheus metrics.
r.GET("/metrics", gin.WrapH(promhttp.Handler()))
-
- // WebSocket 升级端点
+ // WebSocket upgrade (JWT auth via query ?token= or Authorization header).
r.GET("/ws", wsHandler.HandleWebSocket)
-
- // 内部推送 API（Msg 服务调用）
- api := r.Group("/internal")
- api.POST("/push", wsHandler.PushHandler)
- api.POST("/broadcast", wsHandler.BroadcastHandler)
+ // Internal HTTP APIs consumed by msg (X-Internal-Token auth).
+ internal := r.Group("/internal")
+ internal.POST("/push", wsHandler.PushHandler)
+ internal.POST("/broadcast", wsHandler.BroadcastHandler)
+ internal.GET("/online/:userID", wsHandler.OnlineHandler)
srv := &http.Server{
Addr: ":" + cfg.Port,
Handler: r,
ReadTimeout: 10 * time.Second,
- WriteTimeout: 60 * time.Second, // WebSocket 长连接
+ WriteTimeout: 0, // WebSocket connections are long-lived; no write timeout.
}
+ // 11. Start HTTP server.
go func() {
- log.Printf("Push Gateway listening on :%s", cfg.Port)
- if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
- log.Fatalf("listen: %s\n", err)
+ logger.Info("http server listening", "addr", srv.Addr)
+ if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+ logger.Error("http server fatal error", "err", err)
+ os.Exit(1)
}
}()
+ // 12. Wait for termination signal.
quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
- <-quit
- log.Println("Shutting down Push Gateway...")
+ sig := <-quit
+ logger.Info("shutdown signal received", "signal", sig.String())
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- if err := srv.Shutdown(ctx); err != nil {
- log.Fatal("Server forced to shutdown:", err)
+ // 13. Graceful shutdown sequence (see package doc).
+ shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout)
+ defer shutdownCancel()
+
+ // 13a. Hub.CloseAll sends close frame 1001 to every live connection.
+ h.CloseAll()
+ logger.Info("hub closeAll complete; draining connections",
+ "active_connections", h.ActiveConnections())
+
+ // 13b. Stop accepting new HTTP requests; drain in-flight.
+ if err := srv.Shutdown(shutdownCtx); err != nil {
+ logger.Error("http server shutdown error", "err", err)
}
+
+ // 13c. Stop Kafka consumer (cancel context + close reader).
+ if kafkaCancel != nil {
+ kafkaCancel()
+ }
+ if kafkaConsumer != nil {
+ if err := kafkaConsumer.Close(); err != nil {
+ logger.Warn("kafka consumer close error", "err", err)
+ }
+ }
+
+ // 13d. Stop Redis Pub/Sub subscriber + close client.
+ if pubsubCancel != nil {
+ if err := pubsubCancel(); err != nil {
+ logger.Warn("redis pubsub cancel error", "err", err)
+ }
+ }
+ if redisClient != nil {
+ if err := redisClient.Close(); err != nil {
+ logger.Warn("redis close error", "err", err)
+ }
+ }
+
+ logger.Info("push-gateway shutdown complete")
}