feat(infra): p6 hardening - observability and deploy compose

- 5 NestJS services add /metrics endpoint via app.getHttpAdapter() - prometheus.yml scales to 8 services with rule_files and alertmanager - monitoring compose replaces blackbox with Loki+Promtail - Grafana datasource adds Loki - docker-compose.deploy.yml scales to 11 services - deploy.env.example completes Neo4j/ES/ClickHouse/LLM vars - teacher-bff adds health.controller - CI removes continue-on-error on lint step - teacher-portal lint script changed to eslint src
2026-07-09 10:21:06 +08:00
parent 3ca654619f
commit 566060fade
21 changed files with 554 additions and 59 deletions
--- a/infra/deploy.env.example
+++ b/infra/deploy.env.example
@@ -27,12 +27,29 @@ JWT_AUDIENCE=next-edu-cloud
 API_GATEWAY_PORT=8080
 TEACHER_PORTAL_PORT=3000

-# ============ Kafka（P3 阶段启用，P1 留空）============
+# ============ Kafka（P3+ 启用，留空则 Outbox publisher 持续重试）============
 KAFKA_BROKERS=

-# ============ 可观测性（P6 阶段启用，P1 留空）============
-OTEL_EXPORTER_OTLP_ENDPOINT=
+# ============ 可观测性（P6 启用）============
+# OTLP collector 端点，留空则服务跳过 trace 上报
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
 LOG_LEVEL=info

-# ============ 镜像 tag（由 CI 注入，手动部署时可改）============
-# IMAGE_TAG=latest  ← 默认 latest，CI 通过 export IMAGE_TAG 覆盖
+# ============ Neo4j（content 服务，留空则降级模式）============
+NEO4J_URL=
+NEO4J_PASSWORD=
+
+# ============ Elasticsearch（msg 服务，留空则降级模式）============
+ES_URL=
+
+# ============ ClickHouse（data-ana 服务，留空则降级模式）============
+CLICKHOUSE_HOST=
+CLICKHOUSE_PORT=8123
+CLICKHOUSE_DATABASE=edu_analytics
+CLICKHOUSE_USER=
+CLICKHOUSE_PASSWORD=
+
+# ============ LLM 配置（ai 服务，留空则降级模式）============
+OPENAI_API_KEY=
+OPENAI_BASE_URL=https://api.openai.com/v1
+ANTHROPIC_API_KEY=
--- a/infra/docker-compose.deploy.yml
+++ b/infra/docker-compose.deploy.yml
@@ -18,6 +18,9 @@
 name: edu

 services:
+  # ============================================================
+  # 应用服务（10 个：api-gateway + 3 Go/Python + 6 NestJS）
+  # ============================================================
  api-gateway:
    build:
      context: ./repo/services/api-gateway
@@ -34,6 +37,11 @@ services:
      CLASSES_SERVICE_URL: http://classes:3001
      IAM_SERVICE_URL: http://iam:3002
      TEACHER_BFF_URL: http://teacher-bff:3003
+      CORE_EDU_SERVICE_URL: http://core-edu:3004
+      CONTENT_SERVICE_URL: http://content:3005
+      DATA_ANA_SERVICE_URL: http://data-ana:3006
+      MSG_SERVICE_URL: http://msg:3007
+      AI_SERVICE_URL: http://ai:3008
      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
      LOG_LEVEL: ${LOG_LEVEL:-info}
    ports:
@@ -59,7 +67,6 @@ services:
    restart: unless-stopped
    environment:
      PORT: 3001
-      # 连接服务器已有的 MySQL（容器名 edu-mysql，需在 edu-shared 网络）
      DATABASE_URL: ${DATABASE_URL}
      REDIS_URL: ${REDIS_URL}
      KAFKA_BROKERS: ${KAFKA_BROKERS:-}
@@ -75,6 +82,214 @@ services:
      - edu-net
      - edu-shared

+  iam:
+    build:
+      context: ./repo
+      dockerfile: services/iam/Dockerfile
+    container_name: edu-iam
+    restart: unless-stopped
+    environment:
+      PORT: 3002
+      DATABASE_URL: ${DATABASE_URL}
+      REDIS_URL: ${REDIS_URL}
+      JWT_SECRET: ${JWT_SECRET}
+      JWT_ISSUER: ${JWT_ISSUER:-next-edu-cloud}
+      JWT_AUDIENCE: ${JWT_AUDIENCE:-next-edu-cloud}
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      NODE_ENV: production
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--spider", "http://localhost:3002/healthz"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
+    networks:
+      - edu-net
+      - edu-shared
+
+  teacher-bff:
+    build:
+      context: ./repo
+      dockerfile: services/teacher-bff/Dockerfile
+    container_name: edu-teacher-bff
+    restart: unless-stopped
+    environment:
+      PORT: 3003
+      IAM_SERVICE_URL: http://iam:3002
+      CLASSES_SERVICE_URL: http://classes:3001
+      CORE_EDU_SERVICE_URL: http://core-edu:3004
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      NODE_ENV: production
+    depends_on:
+      iam:
+        condition: service_healthy
+      classes:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--spider", "http://localhost:3003/healthz"]
+      interval: 30s
+      timeout: 5s
+      start_period: 20s
+      retries: 3
+    networks:
+      - edu-net
+      - edu-shared
+
+  core-edu:
+    build:
+      context: ./repo
+      dockerfile: services/core-edu/Dockerfile
+    container_name: edu-core-edu
+    restart: unless-stopped
+    environment:
+      PORT: 3004
+      DATABASE_URL: ${DATABASE_URL}
+      REDIS_URL: ${REDIS_URL}
+      KAFKA_BROKERS: ${KAFKA_BROKERS:-}
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      NODE_ENV: production
+      DEV_MODE: "false"
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--spider", "http://localhost:3004/healthz"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
+    networks:
+      - edu-net
+      - edu-shared
+
+  content:
+    build:
+      context: ./repo
+      dockerfile: services/content/Dockerfile
+    container_name: edu-content
+    restart: unless-stopped
+    environment:
+      PORT: 3005
+      DATABASE_URL: ${DATABASE_URL}
+      REDIS_URL: ${REDIS_URL}
+      NEO4J_URL: ${NEO4J_URL:-}
+      NEO4J_PASSWORD: ${NEO4J_PASSWORD:-}
+      ES_URL: ${ES_URL:-}
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      NODE_ENV: production
+      DEV_MODE: "false"
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--spider", "http://localhost:3005/healthz"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
+    networks:
+      - edu-net
+      - edu-shared
+
+  msg:
+    build:
+      context: ./repo
+      dockerfile: services/msg/Dockerfile
+    container_name: edu-msg
+    restart: unless-stopped
+    environment:
+      PORT: 3007
+      DATABASE_URL: ${DATABASE_URL}
+      REDIS_URL: ${REDIS_URL}
+      KAFKA_BROKERS: ${KAFKA_BROKERS:-}
+      ES_URL: ${ES_URL:-}
+      PUSH_GATEWAY_URL: http://push-gateway:8081
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      NODE_ENV: production
+      DEV_MODE: "false"
+    depends_on:
+      push-gateway:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--spider", "http://localhost:3007/healthz"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
+    networks:
+      - edu-net
+      - edu-shared
+
+  ai:
+    build:
+      context: ./repo/services/ai
+      dockerfile: Dockerfile
+    container_name: edu-ai
+    restart: unless-stopped
+    environment:
+      PORT: 3008
+      OPENAI_API_KEY: ${OPENAI_API_KEY:-}
+      OPENAI_BASE_URL: ${OPENAI_BASE_URL:-https://api.openai.com/v1}
+      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
+      OTEL_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      DEV_MODE: "false"
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3008/healthz')"]
+      interval: 30s
+      timeout: 5s
+      start_period: 20s
+      retries: 3
+    networks:
+      - edu-net
+      - edu-shared
+
+  data-ana:
+    build:
+      context: ./repo/services/data-ana
+      dockerfile: Dockerfile
+    container_name: edu-data-ana
+    restart: unless-stopped
+    environment:
+      PORT: 3006
+      CLICKHOUSE_HOST: ${CLICKHOUSE_HOST:-}
+      CLICKHOUSE_PORT: ${CLICKHOUSE_PORT:-8123}
+      CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-edu_analytics}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-}
+      OTEL_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      DEV_MODE: "false"
+      KAFKA_BROKERS: ${KAFKA_BROKERS:-}
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3006/healthz')"]
+      interval: 30s
+      timeout: 5s
+      start_period: 20s
+      retries: 3
+    networks:
+      - edu-net
+      - edu-shared
+
+  push-gateway:
+    build:
+      context: ./repo/services/push-gateway
+      dockerfile: Dockerfile
+    container_name: edu-push-gateway
+    restart: unless-stopped
+    environment:
+      PUSH_GATEWAY_PORT: 8081
+      JWT_SECRET: ${JWT_SECRET}
+      REDIS_URL: ${REDIS_URL}
+      DEV_MODE: "false"
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--spider", "http://localhost:8081/healthz"]
+      interval: 30s
+      timeout: 5s
+      start_period: 10s
+      retries: 3
+    networks:
+      - edu-net
+      - edu-shared
+
  teacher-portal:
    build:
      context: ./repo
--- a/infra/docker-compose.monitoring.yml
+++ b/infra/docker-compose.monitoring.yml
@@ -14,6 +14,8 @@ volumes:
    name: edu-alertmanager-data
  grafana-data:
    name: edu-grafana-data
+  loki-data:
+    name: edu-loki-data

 services:
  # ============================================================
@@ -104,16 +106,35 @@ services:
      - edu-network

  # ============================================================
-  # blackbox-exporter - 黑盒探测（HTTP / TCP / ICMP）
+  # Loki - 日志聚合（P6 硬化新增）
  # ============================================================
-  blackbox-exporter:
-    image: prom/blackbox-exporter:v0.25.0
-    container_name: edu-blackbox-exporter
+  loki:
+    image: grafana/loki:3.2.1
+    container_name: edu-loki
    profiles: ["monitoring"]
    restart: unless-stopped
+    command: -config.file=/etc/loki/local-config.yaml
    ports:
-      - "9115:9115"
+      - "3100:3100"
    volumes:
-      - ./blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+      - loki-data:/loki
    networks:
      - edu-network
+
+  # ============================================================
+  # Promtail - 日志采集（收集 Docker 容器日志发送到 Loki）
+  # ============================================================
+  promtail:
+    image: grafana/promtail:3.2.1
+    container_name: edu-promtail
+    profiles: ["monitoring"]
+    restart: unless-stopped
+    command: -config.file=/etc/promtail/config.yml
+    volumes:
+      - ./promtail/config.yml:/etc/promtail/config.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    networks:
+      - edu-network
+    depends_on:
+      - loki
--- a/infra/grafana/provisioning/datasources/prometheus.yml
+++ b/infra/grafana/provisioning/datasources/prometheus.yml
@@ -13,3 +13,12 @@ datasources:
      timeInterval: "15s"
      httpMethod: POST
      manageAlerts: false
+
+  - name: Loki
+    type: loki
+    uid: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
+    jsonData:
+      maxLines: 1000
--- a/infra/prometheus.yml
+++ b/infra/prometheus.yml
@@ -2,11 +2,93 @@ global:
  scrape_interval: 15s
  evaluation_interval: 15s

-scrape_configs:
-  - job_name: 'api-gateway'
-    static_configs:
-      - targets: ['host.docker.internal:8080']
+# 告警规则文件
+rule_files:
+  - /etc/prometheus/rules.yml

+# 告警管理器
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+scrape_configs:
+  # ============================================================
+  # 应用服务（NestJS / FastAPI 暴露 /metrics）
+  # ============================================================
  - job_name: 'classes-service'
    static_configs:
-      - targets: ['host.docker.internal:3001']
+      - targets: ['host.docker.internal:3001']
+        labels:
+          service: classes
+
+  - job_name: 'iam-service'
+    static_configs:
+      - targets: ['host.docker.internal:3002']
+        labels:
+          service: iam
+
+  - job_name: 'teacher-bff'
+    static_configs:
+      - targets: ['host.docker.internal:3003']
+        labels:
+          service: teacher-bff
+
+  - job_name: 'core-edu-service'
+    static_configs:
+      - targets: ['host.docker.internal:3004']
+        labels:
+          service: core-edu
+
+  - job_name: 'content-service'
+    static_configs:
+      - targets: ['host.docker.internal:3005']
+        labels:
+          service: content
+
+  - job_name: 'data-ana-service'
+    static_configs:
+      - targets: ['host.docker.internal:3006']
+        labels:
+          service: data-ana
+
+  - job_name: 'msg-service'
+    static_configs:
+      - targets: ['host.docker.internal:3007']
+        labels:
+          service: msg
+
+  - job_name: 'ai-service'
+    static_configs:
+      - targets: ['host.docker.internal:3008']
+        labels:
+          service: ai
+
+  # ============================================================
+  # 基础设施（docker-compose.minimal/minimal.override）
+  # ============================================================
+  - job_name: 'mysql'
+    static_configs:
+      - targets: ['host.docker.internal:9104']
+        labels:
+          service: mysql
+
+  - job_name: 'redis'
+    static_configs:
+      - targets: ['host.docker.internal:9121']
+        labels:
+          service: redis
+
+  # ============================================================
+  # 监控栈自身
+  # ============================================================
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node-exporter'
+    static_configs:
+      - targets: ['node-exporter:9100']
+        labels:
+          service: node-exporter
--- a/infra/promtail/config.yml
+++ b/infra/promtail/config.yml
@@ -0,0 +1,28 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  # 采集 Docker 容器日志
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    relabel_configs:
+      # 只采集带 edu- 前缀的容器日志
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(edu-.*).*'
+        action: keep
+      # 提取容器名作为 label
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: container_name
+      # 提取镜像名
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: stream