Edu/services/push-gateway/internal/health/readyz.go

// Package health implements the /healthz (liveness) and /readyz (readiness)
// endpoints for push-gateway.
//
// /healthz is a trivial liveness probe: it returns 200 + {status:ok} as long
// as the process is running and the Gin router is serving.
//
// /readyz reports readiness based on downstream dependency health. Per
// ARB-015 §17.4 / ISSUE-058 the readiness probe uses SOFT FAILURE semantics:
// when Redis or Kafka is unavailable the endpoint still returns HTTP 200 but
// carries `degraded: true` plus a `dependencies` block describing which
// component failed. This prevents Kubernetes from evicting the pod when a
// transient dependency blip occurs, at the cost of accepting some degraded
// behavior (no cross-instance fanout, no Kafka consumption) during the blip.
// Only the Hub being in a shutting-down state returns a non-200 (503).
//
// The hard-failure case is limited to:
//   - Hub.closing == true (process is shutting down) -> 503
//   - Internal misconfiguration (both Redis and Kafka missing in non-DevMode)
//     -> still 200 + degraded, since the pod can still serve local WebSocket
//     traffic.
package health

import (
	"context"
	"net/http"
	"time"

	"github.com/edu-cloud/push-gateway/internal/hub"
	"github.com/edu-cloud/push-gateway/internal/kafkaconsumer"
	"github.com/edu-cloud/push-gateway/internal/redisclient"
	"github.com/gin-gonic/gin"
)

// probeTimeout bounds each dependency check so a hung dependency cannot stall
// the readiness probe. Kubernetes typically expects /readyz to respond within
// 5s; we use 1s per probe to leave headroom.
const probeTimeout = 1 * time.Second

// dependencyStatus is the per-component health snapshot included in the
// /readyz response body.
type dependencyStatus struct {
	Ok      bool   `json:"ok"`
	Error   string `json:"error,omitempty"`
	Latency int64  `json:"latency_ms,omitempty"`
}

// readyzResponse is the ActionState-shaped envelope returned by /readyz.
// `degraded` is true when at least one non-critical dependency is unhealthy.
type readyzResponse struct {
	Status       string                      `json:"status"`
	Service      string                      `json:"service"`
	InstanceID   string                      `json:"instance_id"`
	Degraded     bool                        `json:"degraded"`
	Connections  int                         `json:"connections"`
	Users        int                         `json:"users"`
	Dependencies map[string]*dependencyStatus `json:"dependencies"`
}

// Healthz returns a trivial liveness handler: 200 + {status:ok, service}.
// Liveness never depends on downstream components — if the process can serve
// the request, it is alive.
func Healthz(service string) gin.HandlerFunc {
	return func(c *gin.Context) {
		c.JSON(http.StatusOK, gin.H{
			"status":  "ok",
			"service": service,
		})
	}
}

// Readyzer builds the /readyz handler. It probes Redis (PING) and Kafka
// (reader lag / connectivity) and reports degraded state per ARB-015 §17.4.
//
// The Hub is used to report local connection counts and to detect the
// shutting-down state (which triggers a hard 503). The redisClient may be nil
// in DevMode; the kafkaConsumer may be nil when KAFKA_BROKERS is unset. Both
// nil cases are reported as degraded rather than failing the probe.
type Readyzer struct {
	hub      *hub.Hub
	redis    *redisclient.Client
	kafka    *kafkaconsumer.Consumer
	instance string
	service  string
}

// NewReadyzer constructs a Readyzer. redis and kafka may be nil; the resulting
// probe will mark the missing dependency as degraded.
func NewReadyzer(h *hub.Hub, r *redisclient.Client, k *kafkaconsumer.Consumer, service, instanceID string) *Readyzer {
	return &Readyzer{
		hub:      h,
		redis:    r,
		kafka:    k,
		instance: instanceID,
		service:  service,
	}
}

// Handler is the gin.HandlerFunc for GET /readyz.
func (rz *Readyzer) Handler(c *gin.Context) {
	// Hard failure: Hub is shutting down — reject new traffic.
	// (Hub.CloseAll sets closing=true; we treat this as 503 so the load
	// balancer stops sending WebSocket upgrades during drain.)
	if rz.hub.IsClosing() {
		c.JSON(http.StatusServiceUnavailable, readyzResponse{
			Status:      "shutting_down",
			Service:     rz.service,
			InstanceID:  rz.instance,
			Degraded:    true,
			Connections: rz.hub.ActiveConnections(),
			Users:       rz.hub.UserCount(),
			Dependencies: map[string]*dependencyStatus{
				"hub": {Ok: false, Error: "hub is closing"},
			},
		})
		return
	}

	deps := make(map[string]*dependencyStatus, 2)
	degraded := false

	// Redis probe (soft failure).
	if rz.redis == nil {
		deps["redis"] = &dependencyStatus{Ok: false, Error: "not configured"}
		degraded = true
	} else {
		deps["redis"] = probeRedis(rz.redis)
		if !deps["redis"].Ok {
			degraded = true
		}
	}

	// Kafka probe (soft failure).
	if rz.kafka == nil {
		deps["kafka"] = &dependencyStatus{Ok: false, Error: "not configured"}
		degraded = true
	} else {
		deps["kafka"] = probeKafka(rz.kafka)
		if !deps["kafka"].Ok {
			degraded = true
		}
	}

	// Always 200 (unless shutting down) per ISSUE-058/006 soft-failure rule.
	c.JSON(http.StatusOK, readyzResponse{
		Status:       statusText(degraded),
		Service:      rz.service,
		InstanceID:   rz.instance,
		Degraded:     degraded,
		Connections:  rz.hub.ActiveConnections(),
		Users:        rz.hub.UserCount(),
		Dependencies: deps,
	})
}

// statusText returns "ok" when healthy, "degraded" when at least one
// dependency is unhealthy.
func statusText(degraded bool) string {
	if degraded {
		return "degraded"
	}
	return "ok"
}

// probeRedis issues a PING with a short timeout and reports latency.
func probeRedis(r *redisclient.Client) *dependencyStatus {
	ctx, cancel := context.WithTimeout(context.Background(), probeTimeout)
	defer cancel()
	start := time.Now()
	if err := r.Ping(ctx); err != nil {
		return &dependencyStatus{Ok: false, Error: err.Error()}
	}
	return &dependencyStatus{Ok: true, Latency: time.Since(start).Milliseconds()}
}

// probeKafka checks that the consumer reader is still reachable. We use a
// lightweight Lag() call (segmentio/kafka-go Client API); on failure the
// consumer is marked degraded. Note: a degraded Kafka does NOT block WebSocket
// traffic — it only pauses notification consumption until recovery.
func probeKafka(k *kafkaconsumer.Consumer) *dependencyStatus {
	ctx, cancel := context.WithTimeout(context.Background(), probeTimeout)
	defer cancel()
	if err := k.HealthCheck(ctx); err != nil {
		return &dependencyStatus{Ok: false, Error: err.Error()}
	}
	return &dependencyStatus{Ok: true}
}