chore(push-gateway): ai02 module updates - config, hub, ws, kafka, health, docs

services/push-gateway/internal/health/readyz.go

@@ -0,0 +1,186 @@
+// Package health implements the /healthz (liveness) and /readyz (readiness)
+// endpoints for push-gateway.
+//
+// /healthz is a trivial liveness probe: it returns 200 + {status:ok} as long
+// as the process is running and the Gin router is serving.
+//
+// /readyz reports readiness based on downstream dependency health. Per
+// ARB-015 §17.4 / ISSUE-058 the readiness probe uses SOFT FAILURE semantics:
+// when Redis or Kafka is unavailable the endpoint still returns HTTP 200 but
+// carries `degraded: true` plus a `dependencies` block describing which
+// component failed. This prevents Kubernetes from evicting the pod when a
+// transient dependency blip occurs, at the cost of accepting some degraded
+// behavior (no cross-instance fanout, no Kafka consumption) during the blip.
+// Only the Hub being in a shutting-down state returns a non-200 (503).
+//
+// The hard-failure case is limited to:
+//   - Hub.closing == true (process is shutting down) -> 503
+//   - Internal misconfiguration (both Redis and Kafka missing in non-DevMode)
+//     -> still 200 + degraded, since the pod can still serve local WebSocket
+//     traffic.
+package health
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/edu-cloud/push-gateway/internal/hub"
+	"github.com/edu-cloud/push-gateway/internal/kafkaconsumer"
+	"github.com/edu-cloud/push-gateway/internal/redisclient"
+	"github.com/gin-gonic/gin"
+)
+
+// probeTimeout bounds each dependency check so a hung dependency cannot stall
+// the readiness probe. Kubernetes typically expects /readyz to respond within
+// 5s; we use 1s per probe to leave headroom.
+const probeTimeout = 1 * time.Second
+
+// dependencyStatus is the per-component health snapshot included in the
+// /readyz response body.
+type dependencyStatus struct {
+	Ok      bool   `json:"ok"`
+	Error   string `json:"error,omitempty"`
+	Latency int64  `json:"latency_ms,omitempty"`
+}
+
+// readyzResponse is the ActionState-shaped envelope returned by /readyz.
+// `degraded` is true when at least one non-critical dependency is unhealthy.
+type readyzResponse struct {
+	Status       string                      `json:"status"`
+	Service      string                      `json:"service"`
+	InstanceID   string                      `json:"instance_id"`
+	Degraded     bool                        `json:"degraded"`
+	Connections  int                         `json:"connections"`
+	Users        int                         `json:"users"`
+	Dependencies map[string]*dependencyStatus `json:"dependencies"`
+}
+
+// Healthz returns a trivial liveness handler: 200 + {status:ok, service}.
+// Liveness never depends on downstream components — if the process can serve
+// the request, it is alive.
+func Healthz(service string) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.JSON(http.StatusOK, gin.H{
+			"status":  "ok",
+			"service": service,
+		})
+	}
+}
+
+// Readyzer builds the /readyz handler. It probes Redis (PING) and Kafka
+// (reader lag / connectivity) and reports degraded state per ARB-015 §17.4.
+//
+// The Hub is used to report local connection counts and to detect the
+// shutting-down state (which triggers a hard 503). The redisClient may be nil
+// in DevMode; the kafkaConsumer may be nil when KAFKA_BROKERS is unset. Both
+// nil cases are reported as degraded rather than failing the probe.
+type Readyzer struct {
+	hub      *hub.Hub
+	redis    *redisclient.Client
+	kafka    *kafkaconsumer.Consumer
+	instance string
+	service  string
+}
+
+// NewReadyzer constructs a Readyzer. redis and kafka may be nil; the resulting
+// probe will mark the missing dependency as degraded.
+func NewReadyzer(h *hub.Hub, r *redisclient.Client, k *kafkaconsumer.Consumer, service, instanceID string) *Readyzer {
+	return &Readyzer{
+		hub:      h,
+		redis:    r,
+		kafka:    k,
+		instance: instanceID,
+		service:  service,
+	}
+}
+
+// Handler is the gin.HandlerFunc for GET /readyz.
+func (rz *Readyzer) Handler(c *gin.Context) {
+	// Hard failure: Hub is shutting down — reject new traffic.
+	// (Hub.CloseAll sets closing=true; we treat this as 503 so the load
+	// balancer stops sending WebSocket upgrades during drain.)
+	if rz.hub.IsClosing() {
+		c.JSON(http.StatusServiceUnavailable, readyzResponse{
+			Status:      "shutting_down",
+			Service:     rz.service,
+			InstanceID:  rz.instance,
+			Degraded:    true,
+			Connections: rz.hub.ActiveConnections(),
+			Users:       rz.hub.UserCount(),
+			Dependencies: map[string]*dependencyStatus{
+				"hub": {Ok: false, Error: "hub is closing"},
+			},
+		})
+		return
+	}
+
+	deps := make(map[string]*dependencyStatus, 2)
+	degraded := false
+
+	// Redis probe (soft failure).
+	if rz.redis == nil {
+		deps["redis"] = &dependencyStatus{Ok: false, Error: "not configured"}
+		degraded = true
+	} else {
+		deps["redis"] = probeRedis(rz.redis)
+		if !deps["redis"].Ok {
+			degraded = true
+		}
+	}
+
+	// Kafka probe (soft failure).
+	if rz.kafka == nil {
+		deps["kafka"] = &dependencyStatus{Ok: false, Error: "not configured"}
+		degraded = true
+	} else {
+		deps["kafka"] = probeKafka(rz.kafka)
+		if !deps["kafka"].Ok {
+			degraded = true
+		}
+	}
+
+	// Always 200 (unless shutting down) per ISSUE-058/006 soft-failure rule.
+	c.JSON(http.StatusOK, readyzResponse{
+		Status:       statusText(degraded),
+		Service:      rz.service,
+		InstanceID:   rz.instance,
+		Degraded:     degraded,
+		Connections:  rz.hub.ActiveConnections(),
+		Users:        rz.hub.UserCount(),
+		Dependencies: deps,
+	})
+}
+
+// statusText returns "ok" when healthy, "degraded" when at least one
+// dependency is unhealthy.
+func statusText(degraded bool) string {
+	if degraded {
+		return "degraded"
+	}
+	return "ok"
+}
+
+// probeRedis issues a PING with a short timeout and reports latency.
+func probeRedis(r *redisclient.Client) *dependencyStatus {
+	ctx, cancel := context.WithTimeout(context.Background(), probeTimeout)
+	defer cancel()
+	start := time.Now()
+	if err := r.Ping(ctx); err != nil {
+		return &dependencyStatus{Ok: false, Error: err.Error()}
+	}
+	return &dependencyStatus{Ok: true, Latency: time.Since(start).Milliseconds()}
+}
+
+// probeKafka checks that the consumer reader is still reachable. We use a
+// lightweight Lag() call (segmentio/kafka-go Client API); on failure the
+// consumer is marked degraded. Note: a degraded Kafka does NOT block WebSocket
+// traffic — it only pauses notification consumption until recovery.
+func probeKafka(k *kafkaconsumer.Consumer) *dependencyStatus {
+	ctx, cancel := context.WithTimeout(context.Background(), probeTimeout)
+	defer cancel()
+	if err := k.HealthCheck(ctx); err != nil {
+		return &dependencyStatus{Ok: false, Error: err.Error()}
+	}
+	return &dependencyStatus{Ok: true}
+}