feat(push-gateway,msg): redis pubsub backplane for real-time notifications
M7: ADR-040 Redis Pub/Sub as state routing backplane
- push-gateway: remove Kafka consumer, add SSE endpoint
- SSE: subscribe to Redis user:{userId}:notify on connect
- msg: publish notifications to Redis Pub/Sub instead of HTTP push
- docker-compose: remove Kafka env from push-gateway
This commit is contained in:
SpecialX
@@ -10,7 +10,6 @@ require (
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/redis/go-redis/v9 v9.7.0
|
||||
github.com/segmentio/kafka-go v0.4.48
|
||||
go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.69.0
|
||||
go.opentelemetry.io/otel/trace v1.44.0
|
||||
)
|
||||
@@ -35,7 +34,6 @@ require (
|
||||
github.com/goccy/go-yaml v1.19.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||
@@ -43,7 +41,6 @@ require (
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.3.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
|
||||
@@ -59,7 +59,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF2
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
@@ -83,8 +82,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc=
|
||||
github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
@@ -103,8 +100,6 @@ github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa
|
||||
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs=
|
||||
github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
@@ -120,14 +115,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY=
|
||||
github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
|
||||
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8=
|
||||
go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
@@ -162,53 +149,14 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU=
|
||||
golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8=
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
// variables. Required variables (JWT_SECRET in production, PUSH_INTERNAL_TOKEN
|
||||
// or INTERNAL_API_TOKEN when not in DevMode) cause a fatal exit when missing.
|
||||
//
|
||||
// v2 alignment (ARB-013 / msg contract): Kafka topic default changed to
|
||||
// edu.notify.notification.sent; PUSH_INTERNAL_TOKEN is the canonical env var
|
||||
// for /internal/* auth (INTERNAL_API_TOKEN kept as a backward-compat alias).
|
||||
// M7 (ADR-040): Kafka config removed — push-gateway no longer mounts Kafka
|
||||
// directly. Real-time push is routed through Redis Pub/Sub backplane
|
||||
// (channel: user:{userId}:notify). PUSH_INTERNAL_TOKEN is the canonical env
|
||||
// var for /internal/* auth (INTERNAL_API_TOKEN kept as a backward-compat
|
||||
// alias).
|
||||
package config
|
||||
|
||||
import (
|
||||
@@ -29,10 +31,6 @@ type Config struct {
|
||||
HeartbeatInterval int // seconds
|
||||
// JWT RS256 (via shared-go/jwks, iam /.well-known/jwks.json)
|
||||
JWKSURL string
|
||||
// Kafka
|
||||
KafkaBrokers []string
|
||||
KafkaNotificationTopic string
|
||||
KafkaConsumerGroup string
|
||||
// Instance identity (for Redis SET membership)
|
||||
InstanceID string
|
||||
}
|
||||
@@ -80,11 +78,6 @@ func Load() *Config {
|
||||
MaxConnsPerUser: env.GetInt("MAX_CONNS_PER_USER", 5),
|
||||
HeartbeatInterval: env.GetInt("HEARTBEAT_INTERVAL_SECONDS", 30),
|
||||
JWKSURL: env.Get("JWKS_URL", "http://localhost:50052/.well-known/jwks.json"),
|
||||
KafkaBrokers: parseBrokers(env.Get("KAFKA_BROKERS", "localhost:9092")),
|
||||
// ARB-013 canonical topic name: edu.<domain>.<aggregate>.<action>.
|
||||
// msg's Outbox publisher emits to this topic.
|
||||
KafkaNotificationTopic: env.Get("KAFKA_NOTIFICATION_TOPIC", "edu.notify.notification.sent"),
|
||||
KafkaConsumerGroup: env.Get("KAFKA_CONSUMER_GROUP", "push-gateway"),
|
||||
InstanceID: env.Get("INSTANCE_ID", generateInstanceID()),
|
||||
}
|
||||
}
|
||||
@@ -105,18 +98,6 @@ func parseOrigins(raw string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// parseBrokers splits a comma-separated list of Kafka broker addresses.
|
||||
func parseBrokers(raw string) []string {
|
||||
parts := strings.Split(raw, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
if trimmed := strings.TrimSpace(p); trimmed != "" {
|
||||
out = append(out, trimmed)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// generateInstanceID returns a stable process-unique identifier. Falls back to
|
||||
// the hostname when INSTANCE_ID is not explicitly set, allowing each replica to
|
||||
// be uniquely identifiable in the Redis online-presence SET.
|
||||
|
||||
@@ -32,13 +32,6 @@ func TestLoadDevMode(t *testing.T) {
|
||||
if cfg.HeartbeatInterval != 30 {
|
||||
t.Errorf("HeartbeatInterval = %d, want 30", cfg.HeartbeatInterval)
|
||||
}
|
||||
// ARB-013: canonical topic name is edu.notify.notification.sent.
|
||||
if cfg.KafkaNotificationTopic != "edu.notify.notification.sent" {
|
||||
t.Errorf("KafkaNotificationTopic = %q, want edu.notify.notification.sent", cfg.KafkaNotificationTopic)
|
||||
}
|
||||
if cfg.KafkaConsumerGroup != "push-gateway" {
|
||||
t.Errorf("KafkaConsumerGroup = %q, want push-gateway", cfg.KafkaConsumerGroup)
|
||||
}
|
||||
if cfg.InstanceID == "" {
|
||||
t.Error("InstanceID should default to hostname, got empty")
|
||||
}
|
||||
@@ -113,17 +106,6 @@ func TestParseOrigins(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseBrokers verifies comma-separated broker parsing.
|
||||
func TestParseBrokers(t *testing.T) {
|
||||
got := parseBrokers("kafka1:9092,kafka2:9092,kafka3:9092")
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("parseBrokers returned %d items, want 3", len(got))
|
||||
}
|
||||
if got[0] != "kafka1:9092" || got[1] != "kafka2:9092" || got[2] != "kafka3:9092" {
|
||||
t.Errorf("parseBrokers = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGenerateInstanceID verifies fallback to hostname.
|
||||
func TestGenerateInstanceID(t *testing.T) {
|
||||
id := generateInstanceID()
|
||||
|
||||
@@ -6,18 +6,19 @@
|
||||
//
|
||||
// /readyz reports readiness based on downstream dependency health. Per
|
||||
// ARB-015 §17.4 / ISSUE-058 the readiness probe uses SOFT FAILURE semantics:
|
||||
// when Redis or Kafka is unavailable the endpoint still returns HTTP 200 but
|
||||
// carries `degraded: true` plus a `dependencies` block describing which
|
||||
// component failed. This prevents Kubernetes from evicting the pod when a
|
||||
// transient dependency blip occurs, at the cost of accepting some degraded
|
||||
// behavior (no cross-instance fanout, no Kafka consumption) during the blip.
|
||||
// Only the Hub being in a shutting-down state returns a non-200 (503).
|
||||
// when Redis is unavailable the endpoint still returns HTTP 200 but carries
|
||||
// `degraded: true` plus a `dependencies` block describing which component
|
||||
// failed. This prevents Kubernetes from evicting the pod when a transient
|
||||
// dependency blip occurs, at the cost of accepting some degraded behavior
|
||||
// (no Redis Pub/Sub fanout, no SSE delivery) during the blip. Only the Hub
|
||||
// being in a shutting-down state returns a non-200 (503).
|
||||
//
|
||||
// M7 (ADR-040): Kafka probe removed — push-gateway no longer mounts Kafka.
|
||||
//
|
||||
// The hard-failure case is limited to:
|
||||
// - Hub.closing == true (process is shutting down) -> 503
|
||||
// - Internal misconfiguration (both Redis and Kafka missing in non-DevMode)
|
||||
// -> still 200 + degraded, since the pod can still serve local WebSocket
|
||||
// traffic.
|
||||
// - Redis missing in non-DevMode -> still 200 + degraded, since the pod can
|
||||
// still serve local WebSocket traffic.
|
||||
package health
|
||||
|
||||
import (
|
||||
@@ -26,7 +27,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/edu-cloud/push-gateway/internal/hub"
|
||||
"github.com/edu-cloud/push-gateway/internal/kafkaconsumer"
|
||||
"github.com/edu-cloud/push-gateway/internal/redisclient"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
@@ -47,12 +47,12 @@ type dependencyStatus struct {
|
||||
// readyzResponse is the ActionState-shaped envelope returned by /readyz.
|
||||
// `degraded` is true when at least one non-critical dependency is unhealthy.
|
||||
type readyzResponse struct {
|
||||
Status string `json:"status"`
|
||||
Service string `json:"service"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
Degraded bool `json:"degraded"`
|
||||
Connections int `json:"connections"`
|
||||
Users int `json:"users"`
|
||||
Status string `json:"status"`
|
||||
Service string `json:"service"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
Degraded bool `json:"degraded"`
|
||||
Connections int `json:"connections"`
|
||||
Users int `json:"users"`
|
||||
Dependencies map[string]*dependencyStatus `json:"dependencies"`
|
||||
}
|
||||
|
||||
@@ -68,28 +68,26 @@ func Healthz(service string) gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// Readyzer builds the /readyz handler. It probes Redis (PING) and Kafka
|
||||
// (reader lag / connectivity) and reports degraded state per ARB-015 §17.4.
|
||||
// Readyzer builds the /readyz handler. It probes Redis (PING) and reports
|
||||
// degraded state per ARB-015 §17.4.
|
||||
//
|
||||
// The Hub is used to report local connection counts and to detect the
|
||||
// shutting-down state (which triggers a hard 503). The redisClient may be nil
|
||||
// in DevMode; the kafkaConsumer may be nil when KAFKA_BROKERS is unset. Both
|
||||
// nil cases are reported as degraded rather than failing the probe.
|
||||
// in DevMode; the nil case is reported as degraded rather than failing the
|
||||
// probe.
|
||||
type Readyzer struct {
|
||||
hub *hub.Hub
|
||||
redis *redisclient.Client
|
||||
kafka *kafkaconsumer.Consumer
|
||||
instance string
|
||||
service string
|
||||
}
|
||||
|
||||
// NewReadyzer constructs a Readyzer. redis and kafka may be nil; the resulting
|
||||
// probe will mark the missing dependency as degraded.
|
||||
func NewReadyzer(h *hub.Hub, r *redisclient.Client, k *kafkaconsumer.Consumer, service, instanceID string) *Readyzer {
|
||||
// NewReadyzer constructs a Readyzer. redis may be nil; the resulting probe
|
||||
// will mark the missing dependency as degraded.
|
||||
func NewReadyzer(h *hub.Hub, r *redisclient.Client, service, instanceID string) *Readyzer {
|
||||
return &Readyzer{
|
||||
hub: h,
|
||||
redis: r,
|
||||
kafka: k,
|
||||
instance: instanceID,
|
||||
service: service,
|
||||
}
|
||||
@@ -98,8 +96,6 @@ func NewReadyzer(h *hub.Hub, r *redisclient.Client, k *kafkaconsumer.Consumer, s
|
||||
// Handler is the gin.HandlerFunc for GET /readyz.
|
||||
func (rz *Readyzer) Handler(c *gin.Context) {
|
||||
// Hard failure: Hub is shutting down — reject new traffic.
|
||||
// (Hub.CloseAll sets closing=true; we treat this as 503 so the load
|
||||
// balancer stops sending WebSocket upgrades during drain.)
|
||||
if rz.hub.IsClosing() {
|
||||
c.JSON(http.StatusServiceUnavailable, readyzResponse{
|
||||
Status: "shutting_down",
|
||||
@@ -115,7 +111,7 @@ func (rz *Readyzer) Handler(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
deps := make(map[string]*dependencyStatus, 2)
|
||||
deps := make(map[string]*dependencyStatus, 1)
|
||||
degraded := false
|
||||
|
||||
// Redis probe (soft failure).
|
||||
@@ -129,17 +125,6 @@ func (rz *Readyzer) Handler(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Kafka probe (soft failure).
|
||||
if rz.kafka == nil {
|
||||
deps["kafka"] = &dependencyStatus{Ok: false, Error: "not configured"}
|
||||
degraded = true
|
||||
} else {
|
||||
deps["kafka"] = probeKafka(rz.kafka)
|
||||
if !deps["kafka"].Ok {
|
||||
degraded = true
|
||||
}
|
||||
}
|
||||
|
||||
// Always 200 (unless shutting down) per ISSUE-058/006 soft-failure rule.
|
||||
c.JSON(http.StatusOK, readyzResponse{
|
||||
Status: statusText(degraded),
|
||||
@@ -171,16 +156,3 @@ func probeRedis(r *redisclient.Client) *dependencyStatus {
|
||||
}
|
||||
return &dependencyStatus{Ok: true, Latency: time.Since(start).Milliseconds()}
|
||||
}
|
||||
|
||||
// probeKafka checks that the consumer reader is still reachable. We use a
|
||||
// lightweight Lag() call (segmentio/kafka-go Client API); on failure the
|
||||
// consumer is marked degraded. Note: a degraded Kafka does NOT block WebSocket
|
||||
// traffic — it only pauses notification consumption until recovery.
|
||||
func probeKafka(k *kafkaconsumer.Consumer) *dependencyStatus {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), probeTimeout)
|
||||
defer cancel()
|
||||
if err := k.HealthCheck(ctx); err != nil {
|
||||
return &dependencyStatus{Ok: false, Error: err.Error()}
|
||||
}
|
||||
return &dependencyStatus{Ok: true}
|
||||
}
|
||||
|
||||
@@ -1,253 +0,0 @@
|
||||
// Package kafkaconsumer subscribes to the edu.notify.notification.sent topic
|
||||
// (ARB-013 canonical topic name) on behalf of push-gateway. Each consumed
|
||||
// NotificationRequested event is dispatched to the Hub for delivery to online
|
||||
// clients. At-least-once delivery is enforced via manual commit; idempotency
|
||||
// is provided by Redis SETNX on event_id (TTL 24h).
|
||||
//
|
||||
// Topic naming follows ARB-013 (edu.<domain>.<aggregate>.<action>); the
|
||||
// previous name edu.notification.requested is deprecated (v2 alignment).
|
||||
package kafkaconsumer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/edu-cloud/push-gateway/internal/hub"
|
||||
"github.com/edu-cloud/push-gateway/internal/observability"
|
||||
"github.com/edu-cloud/push-gateway/internal/redisclient"
|
||||
"github.com/segmentio/kafka-go"
|
||||
)
|
||||
|
||||
// MaxRetries is the per-message retry count before sending to the dead-letter
|
||||
// topic. After MaxRetries failures the message is committed (skipped) and
|
||||
// logged; a future P6 enhancement can republish to a DLQ topic.
|
||||
const MaxRetries = 3
|
||||
|
||||
// NotificationRequested is the message shape produced by msg (ai10). Field
|
||||
// names mirror the JSON wire format produced by msg's Outbox publisher.
|
||||
type NotificationRequested struct {
|
||||
EventID string `json:"event_id"`
|
||||
UserID string `json:"user_id"`
|
||||
EventType string `json:"event_type"`
|
||||
Channel string `json:"channel"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
Broadcast bool `json:"broadcast"`
|
||||
OccurredAt int64 `json:"occurred_at"`
|
||||
}
|
||||
|
||||
// Consumer wraps a kafka.Reader for the notification topic.
|
||||
type Consumer struct {
|
||||
reader *kafka.Reader
|
||||
hub *hub.Hub
|
||||
redis *redisclient.Client
|
||||
metrics *observability.Metrics
|
||||
topic string
|
||||
groupID string
|
||||
dlqTopic string
|
||||
}
|
||||
|
||||
// Config configures the Kafka consumer.
|
||||
type Config struct {
|
||||
Brokers []string
|
||||
Topic string
|
||||
GroupID string
|
||||
DLQTopic string
|
||||
}
|
||||
|
||||
// New creates a Consumer for the configured topic and group.
|
||||
func New(cfg Config, h *hub.Hub, r *redisclient.Client, m *observability.Metrics) *Consumer {
|
||||
if cfg.DLQTopic == "" {
|
||||
cfg.DLQTopic = cfg.Topic + ".dlq"
|
||||
}
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.Brokers,
|
||||
Topic: cfg.Topic,
|
||||
GroupID: cfg.GroupID,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10 * 1024 * 1024,
|
||||
CommitInterval: 1 * time.Second, // periodic background commit
|
||||
StartOffset: kafka.LastOffset, // skip historical backlog on first start
|
||||
})
|
||||
return &Consumer{
|
||||
reader: reader,
|
||||
hub: h,
|
||||
redis: r,
|
||||
metrics: m,
|
||||
topic: cfg.Topic,
|
||||
groupID: cfg.GroupID,
|
||||
dlqTopic: cfg.DLQTopic,
|
||||
}
|
||||
}
|
||||
|
||||
// Run blocks until ctx is canceled, consuming messages and dispatching them.
|
||||
// Errors are logged but do not stop the consumer unless ctx is canceled.
|
||||
func (c *Consumer) Run(ctx context.Context) error {
|
||||
observability.Logger().Info("kafka consumer starting",
|
||||
"topic", c.topic, "group", c.groupID, "brokers", c.reader.Config().Brokers)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
msg, err := c.reader.ReadMessage(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
|
||||
return nil
|
||||
}
|
||||
observability.Logger().Warn("kafka read failed", "err", err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
continue
|
||||
}
|
||||
c.metrics.IncKafkaConsumed(c.topic, fmt.Sprintf("%d", msg.Partition))
|
||||
c.processMessage(ctx, msg)
|
||||
}
|
||||
}
|
||||
|
||||
// Close releases the underlying reader. Should be called after Run returns.
|
||||
func (c *Consumer) Close() error {
|
||||
return c.reader.Close()
|
||||
}
|
||||
|
||||
// HealthCheck verifies that the Kafka broker is reachable. Used by /readyz as
|
||||
// a soft-failure probe: a failing check marks the consumer as degraded but
|
||||
// does not return 503 (per ARB-015 §17.4 / ISSUE-058).
|
||||
//
|
||||
// The check issues a metadata request for the consumer topic; on success the
|
||||
// broker is considered reachable. This is cheaper than a Lag() call which
|
||||
// requires partition assignment to have completed.
|
||||
func (c *Consumer) HealthCheck(ctx context.Context) error {
|
||||
conn, err := kafka.DialContext(ctx, "tcp", c.reader.Config().Brokers[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("kafka: dial broker: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
partitions, err := conn.ReadPartitions(c.topic)
|
||||
if err != nil {
|
||||
return fmt.Errorf("kafka: read partitions: %w", err)
|
||||
}
|
||||
if len(partitions) == 0 {
|
||||
return fmt.Errorf("kafka: topic %q has no partitions", c.topic)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Topic returns the configured topic name. Exposed for diagnostics.
|
||||
func (c *Consumer) Topic() string {
|
||||
return c.topic
|
||||
}
|
||||
|
||||
// GroupID returns the configured consumer group name.
|
||||
func (c *Consumer) GroupID() string {
|
||||
return c.groupID
|
||||
}
|
||||
|
||||
// processMessage handles a single Kafka message with retry and idempotency.
|
||||
func (c *Consumer) processMessage(ctx context.Context, msg kafka.Message) {
|
||||
var event NotificationRequested
|
||||
if err := json.Unmarshal(msg.Value, &event); err != nil {
|
||||
observability.Logger().Warn("kafka: invalid message payload",
|
||||
"topic", c.topic, "partition", msg.Partition, "offset", msg.Offset, "err", err)
|
||||
c.metrics.IncPushed("notification.invalid", "invalid")
|
||||
// Commit to skip the poison message.
|
||||
_ = c.reader.CommitMessages(ctx, msg)
|
||||
return
|
||||
}
|
||||
|
||||
// eventLabel is the metrics label for IncPushed. Prefer event_type (the
|
||||
// actual event name like "ExamExtended"); fall back to a generic label.
|
||||
eventLabel := event.EventType
|
||||
if eventLabel == "" {
|
||||
eventLabel = "notification.sent"
|
||||
}
|
||||
|
||||
// Idempotency: skip already-processed events by event_id.
|
||||
fresh, err := c.redis.DedupEventId(ctx, event.EventID)
|
||||
if err != nil {
|
||||
observability.Logger().Warn("kafka: dedup check failed, processing anyway",
|
||||
"event_id", event.EventID, "err", err)
|
||||
}
|
||||
if !fresh {
|
||||
observability.Logger().Debug("kafka: duplicate event skipped",
|
||||
"event_id", event.EventID)
|
||||
c.metrics.IncPushed(eventLabel, "duplicate")
|
||||
_ = c.reader.CommitMessages(ctx, msg)
|
||||
return
|
||||
}
|
||||
|
||||
// Retry up to MaxRetries on dispatch failure.
|
||||
for attempt := 1; attempt <= MaxRetries; attempt++ {
|
||||
if err := c.dispatch(ctx, event); err == nil {
|
||||
c.metrics.IncPushed(eventLabel, "delivered")
|
||||
_ = c.reader.CommitMessages(ctx, msg)
|
||||
return
|
||||
} else if attempt < MaxRetries {
|
||||
observability.Logger().Warn("kafka: dispatch retry",
|
||||
"event_id", event.EventID, "attempt", attempt, "err", err)
|
||||
time.Sleep(time.Duration(attempt) * time.Second)
|
||||
}
|
||||
}
|
||||
// MaxRetries exhausted: send to DLQ (best-effort) and commit.
|
||||
observability.Logger().Error("kafka: dispatch failed after retries, sending to DLQ",
|
||||
"event_id", event.EventID, "topic", c.topic)
|
||||
c.sendToDLQ(ctx, msg.Value)
|
||||
c.metrics.IncPushed(eventLabel, "dlq")
|
||||
_ = c.reader.CommitMessages(ctx, msg)
|
||||
}
|
||||
|
||||
// dispatch delivers the event to local Hub connections and, when the user is
|
||||
// not local, asks Redis to fan out via Pub/Sub.
|
||||
func (c *Consumer) dispatch(ctx context.Context, event NotificationRequested) error {
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"type": "message",
|
||||
"event": event.EventType,
|
||||
"data": event.Data,
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
|
||||
if event.Broadcast {
|
||||
// Broadcast: deliver locally + publish to other instances.
|
||||
c.hub.Broadcast(payload)
|
||||
return c.redis.PublishBroadcast(ctx, redisclient.CrossInstanceMessage{
|
||||
Event: event.EventType,
|
||||
Data: event.Data,
|
||||
})
|
||||
}
|
||||
|
||||
// Directed: try local first, fall back to Redis Pub/Sub.
|
||||
if c.hub.HasUser(event.UserID) {
|
||||
if c.hub.SendToUser(event.UserID, payload) > 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return c.redis.PublishUser(ctx, event.UserID, redisclient.CrossInstanceMessage{
|
||||
Event: event.EventType,
|
||||
Data: event.Data,
|
||||
})
|
||||
}
|
||||
|
||||
// sendToDLQ publishes a raw message to the dead-letter topic. Best-effort;
|
||||
// failure is logged but does not block commit.
|
||||
func (c *Consumer) sendToDLQ(ctx context.Context, value []byte) {
|
||||
conn, err := kafka.DialLeader(ctx, "tcp", c.reader.Config().Brokers[0], c.dlqTopic, 0)
|
||||
if err != nil {
|
||||
observability.Logger().Error("kafka: DLQ dial failed", "topic", c.dlqTopic, "err", err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
_, _ = conn.WriteMessages(kafka.Message{Value: value})
|
||||
}
|
||||
@@ -359,6 +359,16 @@ func idempotentKey(eventID string) string {
|
||||
return "edu:push:idempotent:" + eventID
|
||||
}
|
||||
|
||||
// NotifyChannel returns the Redis Pub/Sub channel name for a user's real-time
|
||||
// notification stream. M7 (ADR-040): msg service publishes notifications to
|
||||
// this channel; realtime-gateway instances subscribe per-connection when a
|
||||
// user connects via SSE.
|
||||
//
|
||||
// Channel naming: `user:{userId}:notify`
|
||||
func NotifyChannel(userID string) string {
|
||||
return "user:" + userID + ":notify"
|
||||
}
|
||||
|
||||
// ParseUserChannel extracts the userID from a "edu:push:channel:user:<userID>"
|
||||
// channel name. Returns empty string when the name does not match.
|
||||
func ParseUserChannel(channel string) string {
|
||||
|
||||
217
services/push-gateway/internal/sse/handler.go
Normal file
217
services/push-gateway/internal/sse/handler.go
Normal file
@@ -0,0 +1,217 @@
|
||||
// Package sse implements the Server-Sent Events endpoint for real-time
|
||||
// notification delivery.
|
||||
//
|
||||
// M7 (ADR-040): The SSE endpoint replaces direct Kafka mounting with a
|
||||
// Redis Pub/Sub backplane. Each SSE connection dynamically subscribes to
|
||||
// the Redis channel `user:{userId}:notify` for the authenticated user.
|
||||
// When the user disconnects, the subscription is released. This per-connection
|
||||
// subscription model means each realtime-gateway instance only holds Redis
|
||||
// subscriptions for users actively connected to it, enabling horizontal
|
||||
// scaling behind a load balancer.
|
||||
//
|
||||
// Authentication:
|
||||
// - JWT RS256 validated via shared-go/jwks (caching JWKS from iam
|
||||
// /.well-known/jwks.json). DevMode accepts the literal "dev-token"
|
||||
// returning a synthetic dev-user subject.
|
||||
//
|
||||
// SSE format: `data: {json}\n\n` (per the HTML5 Server-Sent Events spec).
|
||||
// Heartbeat comments (`: heartbeat\n\n`) are sent every 30s to keep the
|
||||
// connection alive through proxies.
|
||||
package sse
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/edu-cloud/push-gateway/internal/config"
|
||||
"github.com/edu-cloud/push-gateway/internal/observability"
|
||||
"github.com/edu-cloud/push-gateway/internal/redisclient"
|
||||
"github.com/edu-cloud/shared-go/jwks"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
)
|
||||
|
||||
// heartbeatInterval is how often a comment frame is sent to keep the SSE
|
||||
// connection alive through proxies that idle-timeout connections.
|
||||
const heartbeatInterval = 30 * time.Second
|
||||
|
||||
// errResponse is the ActionState-shaped error envelope returned for auth
|
||||
// failures and Redis unavailability.
|
||||
type errResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error *errBody `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type errBody struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
var (
|
||||
errMissingToken = errors.New("missing token")
|
||||
errInvalidToken = errors.New("invalid token")
|
||||
errMissingUserID = errors.New("missing user id")
|
||||
)
|
||||
|
||||
// Handler owns the SSE endpoint for notification delivery.
|
||||
type Handler struct {
|
||||
cfg *config.Config
|
||||
redis *redisclient.Client
|
||||
jwksFetcher *jwks.Fetcher
|
||||
metrics *observability.Metrics
|
||||
}
|
||||
|
||||
// NewHandler constructs an SSE Handler. cfg supplies JWT auth configuration;
|
||||
// redis may be nil in DevMode (SSE returns 503 when Redis is unavailable).
|
||||
// jwksFetcher is initialized from cfg.JWKSURL when non-empty.
|
||||
func NewHandler(cfg *config.Config, r *redisclient.Client, m *observability.Metrics) *Handler {
|
||||
var fetcher *jwks.Fetcher
|
||||
if cfg.JWKSURL != "" {
|
||||
fetcher = jwks.NewFetcher(cfg.JWKSURL)
|
||||
}
|
||||
return &Handler{
|
||||
cfg: cfg,
|
||||
redis: r,
|
||||
jwksFetcher: fetcher,
|
||||
metrics: m,
|
||||
}
|
||||
}
|
||||
|
||||
// HandleNotifications is the GET /sse/notifications endpoint. It upgrades the
|
||||
// HTTP connection to a persistent SSE stream, subscribing to the user's
|
||||
// Redis Pub/Sub channel and forwarding messages as SSE data frames.
|
||||
//
|
||||
// Lifecycle:
|
||||
// 1. Authenticate via JWT (query ?token= or Authorization: Bearer header)
|
||||
// 2. Set SSE headers (Content-Type: text/event-stream, no-cache, etc.)
|
||||
// 3. SUBSCRIBE to Redis channel `user:{userId}:notify`
|
||||
// 4. Forward each Redis message as `data: {json}\n\n`
|
||||
// 5. Send heartbeat comment every 30s
|
||||
// 6. On client disconnect or context cancel: UNSUBSCRIBE (deferred pubsub.Close)
|
||||
func (h *Handler) HandleNotifications(c *gin.Context) {
|
||||
userID, err := h.authenticate(c)
|
||||
if err != nil {
|
||||
c.AbortWithStatusJSON(http.StatusUnauthorized, errResponse{
|
||||
Success: false,
|
||||
Error: &errBody{Code: "SSE_UNAUTHORIZED", Message: err.Error()},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if h.redis == nil {
|
||||
c.AbortWithStatusJSON(http.StatusServiceUnavailable, errResponse{
|
||||
Success: false,
|
||||
Error: &errBody{Code: "SSE_REDIS_UNAVAILABLE", Message: "Redis not configured"},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// SSE response headers.
|
||||
c.Header("Content-Type", "text/event-stream")
|
||||
c.Header("Cache-Control", "no-cache")
|
||||
c.Header("Connection", "keep-alive")
|
||||
c.Header("X-Accel-Buffering", "no") // Disable nginx buffering.
|
||||
|
||||
// Per-connection Redis subscription to the user's notify channel.
|
||||
// go-redis PubSub is goroutine-safe per instance; each Subscribe call
|
||||
// creates an independent PubSub that is closed on connection end.
|
||||
channel := redisclient.NotifyChannel(userID)
|
||||
pubsub := h.redis.Redis().Subscribe(c.Request.Context(), channel)
|
||||
defer pubsub.Close()
|
||||
|
||||
msgCh := pubsub.Channel()
|
||||
ticker := time.NewTicker(heartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
observability.Logger().Info("sse connection opened",
|
||||
"user_id", userID, "channel", channel)
|
||||
|
||||
c.Stream(func(w io.Writer) bool {
|
||||
select {
|
||||
case <-c.Request.Context().Done():
|
||||
return false
|
||||
case msg, ok := <-msgCh:
|
||||
if !ok {
|
||||
// Redis subscription closed (e.g. Redis connection lost).
|
||||
observability.Logger().Warn("sse redis subscription closed",
|
||||
"user_id", userID)
|
||||
return false
|
||||
}
|
||||
// Forward Redis payload as SSE data frame: `data: {json}\n\n`
|
||||
fmt.Fprintf(w, "data: %s\n\n", msg.Payload)
|
||||
h.metrics.IncPushed("sse.notification", "delivered")
|
||||
return true
|
||||
case <-ticker.C:
|
||||
// SSE comment frame keeps the connection alive through proxies.
|
||||
fmt.Fprintf(w, ": heartbeat\n\n")
|
||||
return true
|
||||
}
|
||||
})
|
||||
|
||||
observability.Logger().Info("sse connection closed", "user_id", userID)
|
||||
}
|
||||
|
||||
// authenticate validates the JWT token. Production uses RS256 via shared-go/jwks;
|
||||
// DevMode accepts the literal "dev-token" string. The token may be passed via
|
||||
// the ?token= query parameter or the Authorization: Bearer header.
|
||||
func (h *Handler) authenticate(c *gin.Context) (string, error) {
|
||||
tokenStr := c.Query("token")
|
||||
if tokenStr == "" {
|
||||
if auth := c.GetHeader("Authorization"); strings.HasPrefix(auth, "Bearer ") {
|
||||
tokenStr = strings.TrimPrefix(auth, "Bearer ")
|
||||
}
|
||||
}
|
||||
if tokenStr == "" {
|
||||
return "", errMissingToken
|
||||
}
|
||||
|
||||
// DevMode shortcut: dev-token maps to a synthetic local user.
|
||||
if h.cfg.DevMode && tokenStr == "dev-token" {
|
||||
return "dev-user", nil
|
||||
}
|
||||
|
||||
// RS256 via shared-go/jwks (preferred, requires iam /.well-known/jwks.json).
|
||||
if h.jwksFetcher != nil {
|
||||
claims, err := h.jwksFetcher.ValidateToken(tokenStr)
|
||||
if err != nil {
|
||||
return "", errInvalidToken
|
||||
}
|
||||
if claims.UserID == "" {
|
||||
return "", errMissingUserID
|
||||
}
|
||||
return claims.UserID, nil
|
||||
}
|
||||
|
||||
// DevMode without JWKS: fall back to HS256 with the configured secret.
|
||||
if h.cfg.DevMode && h.cfg.JWTSecret != "" {
|
||||
return validateHS256(tokenStr, h.cfg.JWTSecret)
|
||||
}
|
||||
return "", errInvalidToken
|
||||
}
|
||||
|
||||
// validateHS256 parses and validates an HS256 JWT signed with secret. Used
|
||||
// only in DevMode when no JWKS endpoint is configured (local integration
|
||||
// tests). Returns the user_id claim on success.
|
||||
func validateHS256(tokenStr, secret string) (string, error) {
|
||||
claims := &jwks.Claims{}
|
||||
parsed, err := jwt.ParseWithClaims(tokenStr, claims, func(t *jwt.Token) (any, error) {
|
||||
if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("sse: unexpected signing method %v", t.Header["alg"])
|
||||
}
|
||||
return []byte(secret), nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", errInvalidToken
|
||||
}
|
||||
if !parsed.Valid {
|
||||
return "", errInvalidToken
|
||||
}
|
||||
if claims.UserID == "" {
|
||||
return "", errMissingUserID
|
||||
}
|
||||
return claims.UserID, nil
|
||||
}
|
||||
@@ -1,28 +1,41 @@
|
||||
// Command push-gateway is the Edu platform's real-time WebSocket push gateway.
|
||||
// Command push-gateway is the Edu platform's real-time push gateway.
|
||||
//
|
||||
// It maintains per-user WebSocket connections, delivers directed and broadcast
|
||||
// messages from msg (via HTTP /internal/* APIs and Kafka), and synchronizes
|
||||
// online presence across instances via Redis Pub/Sub.
|
||||
// M7 (ADR-040): The gateway no longer mounts Kafka directly. Real-time push
|
||||
// is routed through a Redis Pub/Sub backplane. The msg service publishes
|
||||
// notifications to Redis channel `user:{userId}:notify` after Kafka
|
||||
// consumption (persistence, desensitization). This gateway dynamically
|
||||
// subscribes to the user's Redis channel when an SSE connection is opened,
|
||||
// and unsubscribes on disconnect.
|
||||
//
|
||||
// Endpoints:
|
||||
// - GET /sse/notifications: SSE stream for real-time notifications (JWT auth,
|
||||
// per-connection Redis subscription). M7 preferred path.
|
||||
// - GET /ws: WebSocket upgrade (JWT auth, legacy path with global Redis
|
||||
// PSubscribe fanout for backwards compatibility).
|
||||
// - POST /internal/push, /internal/broadcast: internal HTTP APIs (X-Internal-Key
|
||||
// auth, consumed by msg for legacy WebSocket fanout).
|
||||
// - GET /internal/online/:userID: presence query.
|
||||
// - GET /healthz, /readyz: health probes.
|
||||
// - GET /metrics: Prometheus metrics.
|
||||
//
|
||||
// Startup sequence:
|
||||
// 1. config.Load (env vars, DevMode detection)
|
||||
// 2. observability.InitLogger (slog JSON/text)
|
||||
// 3. observability.InitTracer (shared-go/tracer → OTLP)
|
||||
// 4. observability.NewMetrics (Prometheus)
|
||||
// 5. hub.NewHub (in-memory connection registry)
|
||||
// 5. hub.NewHub (in-memory connection registry for WebSocket)
|
||||
// 6. redisclient.New + SetHub + RebuildPresenceOnStartup (ISSUE-058)
|
||||
// 7. redisclient.SubscribeAll (cross-instance Pub/Sub fanout)
|
||||
// 8. kafkaconsumer.New + goroutine Run (edu.notify.notification.sent, ARB-013)
|
||||
// 9. ws.NewHandler (JWT RS256, X-Internal-Key, WebSocket upgrade)
|
||||
// 7. redisclient.SubscribeAll (cross-instance Pub/Sub fanout for WebSocket)
|
||||
// 8. ws.NewHandler (JWT RS256, X-Internal-Key, WebSocket upgrade)
|
||||
// 9. sse.NewHandler (JWT RS256, per-connection Redis subscription)
|
||||
// 10. health.NewReadyzer (/readyz soft-failure probe)
|
||||
// 11. gin router + http.Server
|
||||
//
|
||||
// Graceful shutdown (SIGINT/SIGTERM):
|
||||
// 1. Hub.CloseAll (send close frame 1001 to every live connection)
|
||||
// 2. http.Server.Shutdown (stop accepting new HTTP/WebSocket requests)
|
||||
// 3. Kafka consumer goroutine cancel + reader.Close
|
||||
// 4. Redis Pub/Sub cancel + client.Close
|
||||
// 5. Tracer shutdown (flush pending spans)
|
||||
// 1. Hub.CloseAll (send close frame 1001 to every live WebSocket connection)
|
||||
// 2. http.Server.Shutdown (stop accepting new HTTP/SSE/WebSocket requests)
|
||||
// 3. Redis Pub/Sub cancel + client.Close
|
||||
// 4. Tracer shutdown (flush pending spans)
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -37,9 +50,9 @@ import (
|
||||
"github.com/edu-cloud/push-gateway/internal/config"
|
||||
"github.com/edu-cloud/push-gateway/internal/health"
|
||||
"github.com/edu-cloud/push-gateway/internal/hub"
|
||||
"github.com/edu-cloud/push-gateway/internal/kafkaconsumer"
|
||||
"github.com/edu-cloud/push-gateway/internal/observability"
|
||||
"github.com/edu-cloud/push-gateway/internal/redisclient"
|
||||
"github.com/edu-cloud/push-gateway/internal/sse"
|
||||
"github.com/edu-cloud/push-gateway/internal/ws"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
@@ -50,7 +63,8 @@ const serviceName = "push-gateway"
|
||||
|
||||
// shutdownTimeout bounds the graceful shutdown of the HTTP server. After this
|
||||
// window in-flight requests are forcibly closed. WebSocket long-poll
|
||||
// connections are drained by Hub.CloseAll before this timer starts.
|
||||
// connections and SSE streams are drained by Hub.CloseAll before this timer
|
||||
// starts; SSE streams are drained by http.Server.Shutdown.
|
||||
const shutdownTimeout = 10 * time.Second
|
||||
|
||||
func main() {
|
||||
@@ -65,8 +79,6 @@ func main() {
|
||||
"max_conns_per_user", cfg.MaxConnsPerUser,
|
||||
"jwks_url", cfg.JWKSURL,
|
||||
"redis_url", cfg.RedisURL,
|
||||
"kafka_brokers", cfg.KafkaBrokers,
|
||||
"kafka_topic", cfg.KafkaNotificationTopic,
|
||||
)
|
||||
|
||||
// 2. OpenTelemetry tracer (best-effort).
|
||||
@@ -76,7 +88,7 @@ func main() {
|
||||
// 3. Prometheus metrics.
|
||||
metrics := observability.NewMetrics()
|
||||
|
||||
// 4. Hub: in-memory connection registry.
|
||||
// 4. Hub: in-memory connection registry (for WebSocket legacy path).
|
||||
h := hub.NewHub(cfg.MaxConnsPerUser)
|
||||
|
||||
// 5. Redis client (optional in DevMode). When present, wire presence
|
||||
@@ -98,10 +110,10 @@ func main() {
|
||||
logger.Info("redis presence synced", "url", cfg.RedisURL)
|
||||
}
|
||||
} else {
|
||||
logger.Warn("REDIS_URL empty; cross-instance fanout disabled")
|
||||
logger.Warn("REDIS_URL empty; SSE and cross-instance fanout disabled")
|
||||
}
|
||||
|
||||
// 6. Redis Pub/Sub subscriber (cross-instance message fanout).
|
||||
// 6. Redis Pub/Sub subscriber (cross-instance message fanout for WebSocket).
|
||||
var pubsubCancel func() error
|
||||
if redisClient != nil {
|
||||
subCtx, subCancel := context.WithCancel(context.Background())
|
||||
@@ -116,36 +128,14 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Kafka consumer (edu.notify.notification.sent, ARB-013). Started only
|
||||
// when at least one broker is configured; otherwise the HTTP /internal/push
|
||||
// API is the only delivery channel.
|
||||
var kafkaConsumer *kafkaconsumer.Consumer
|
||||
var kafkaCancel context.CancelFunc
|
||||
if len(cfg.KafkaBrokers) > 0 && cfg.KafkaNotificationTopic != "" {
|
||||
kafkaConsumer = kafkaconsumer.New(kafkaconsumer.Config{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
Topic: cfg.KafkaNotificationTopic,
|
||||
GroupID: cfg.KafkaConsumerGroup,
|
||||
}, h, redisClient, metrics)
|
||||
kafkaCtx, cancel := context.WithCancel(context.Background())
|
||||
kafkaCancel = cancel
|
||||
go func() {
|
||||
logger.Info("kafka consumer starting",
|
||||
"topic", cfg.KafkaNotificationTopic, "group", cfg.KafkaConsumerGroup)
|
||||
if err := kafkaConsumer.Run(kafkaCtx); err != nil &&
|
||||
!errors.Is(err, context.Canceled) {
|
||||
logger.Error("kafka consumer exited with error", "err", err)
|
||||
}
|
||||
}()
|
||||
} else {
|
||||
logger.Warn("KAFKA_BROKERS empty; notification consumption disabled")
|
||||
}
|
||||
|
||||
// 8. WebSocket + internal HTTP handlers.
|
||||
// 7. WebSocket + internal HTTP handlers (legacy path, backwards compat).
|
||||
wsHandler := ws.NewHandler(h, cfg, redisClient, metrics)
|
||||
|
||||
// 8. SSE handler (M7 preferred path: per-connection Redis subscription).
|
||||
sseHandler := sse.NewHandler(cfg, redisClient, metrics)
|
||||
|
||||
// 9. /readyz probe (soft failure per ARB-015 §17.4).
|
||||
readyzer := health.NewReadyzer(h, redisClient, kafkaConsumer, serviceName, cfg.InstanceID)
|
||||
readyzer := health.NewReadyzer(h, redisClient, serviceName, cfg.InstanceID)
|
||||
|
||||
// 10. Gin router.
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
@@ -155,10 +145,12 @@ func main() {
|
||||
|
||||
// Liveness (no auth, no dependency checks).
|
||||
r.GET("/healthz", health.Healthz(serviceName))
|
||||
// Readiness (soft failure on Redis/Kafka).
|
||||
// Readiness (soft failure on Redis).
|
||||
r.GET("/readyz", readyzer.Handler)
|
||||
// Prometheus metrics.
|
||||
r.GET("/metrics", gin.WrapH(promhttp.Handler()))
|
||||
// SSE endpoint (M7 preferred: per-connection Redis subscription).
|
||||
r.GET("/sse/notifications", sseHandler.HandleNotifications)
|
||||
// WebSocket upgrade (JWT auth via query ?token= or Authorization header).
|
||||
r.GET("/ws", wsHandler.HandleWebSocket)
|
||||
// Internal HTTP APIs consumed by msg (X-Internal-Key auth, ARB-013).
|
||||
@@ -171,7 +163,7 @@ func main() {
|
||||
Addr: ":" + cfg.Port,
|
||||
Handler: r,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 0, // WebSocket connections are long-lived; no write timeout.
|
||||
WriteTimeout: 0, // SSE and WebSocket connections are long-lived; no write timeout.
|
||||
}
|
||||
|
||||
// 11. Start HTTP server.
|
||||
@@ -193,27 +185,17 @@ func main() {
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
defer shutdownCancel()
|
||||
|
||||
// 13a. Hub.CloseAll sends close frame 1001 to every live connection.
|
||||
// 13a. Hub.CloseAll sends close frame 1001 to every live WebSocket connection.
|
||||
h.CloseAll()
|
||||
logger.Info("hub closeAll complete; draining connections",
|
||||
"active_connections", h.ActiveConnections())
|
||||
|
||||
// 13b. Stop accepting new HTTP requests; drain in-flight.
|
||||
// 13b. Stop accepting new HTTP requests; drain in-flight (including SSE).
|
||||
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||
logger.Error("http server shutdown error", "err", err)
|
||||
}
|
||||
|
||||
// 13c. Stop Kafka consumer (cancel context + close reader).
|
||||
if kafkaCancel != nil {
|
||||
kafkaCancel()
|
||||
}
|
||||
if kafkaConsumer != nil {
|
||||
if err := kafkaConsumer.Close(); err != nil {
|
||||
logger.Warn("kafka consumer close error", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 13d. Stop Redis Pub/Sub subscriber + close client.
|
||||
// 13c. Stop Redis Pub/Sub subscriber + close client.
|
||||
if pubsubCancel != nil {
|
||||
if err := pubsubCancel(); err != nil {
|
||||
logger.Warn("redis pubsub cancel error", "err", err)
|
||||
|
||||
Reference in New Issue
Block a user