Grafana, Prometheus et InfluxDB

# relever les versions courantes sur les pages de publication de chaque projet
PROMETHEUS_VERSION=v3.x.y
ALERTMANAGER_VERSION=v0.x.y
GRAFANA_VERSION=12.x.y
BLACKBOX_VERSION=v0.x.y
PVE_EXPORTER_VERSION=3.x.y
INFLUXDB_IMAGE=influxdb:2

services:
  prometheus:
    image: prom/prometheus:${PROMETHEUS_VERSION}
    restart: unless-stopped
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=90d
      - --storage.tsdb.retention.size=60GB
      - --web.enable-lifecycle
    volumes:
      - ./prometheus:/etc/prometheus:ro
      - prometheus_data:/prometheus
    ports:
      - "127.0.0.1:9090:9090"

  alertmanager:
    image: prom/alertmanager:${ALERTMANAGER_VERSION}
    restart: unless-stopped
    volumes:
      - ./alertmanager:/etc/alertmanager:ro
    ports:
      - "127.0.0.1:9093:9093"

  influxdb:
    image: ${INFLUXDB_IMAGE}
    restart: unless-stopped
    volumes:
      - influxdb_data:/var/lib/influxdb2
    ports:
      # expose sur le reseau d'administration : les noeuds Proxmox y ecrivent
      - "10.10.0.50:8086:8086"

  blackbox:
    image: prom/blackbox-exporter:${BLACKBOX_VERSION}
    restart: unless-stopped

  pve-exporter:
    image: prompve/prometheus-pve-exporter:${PVE_EXPORTER_VERSION}
    restart: unless-stopped
    volumes:
      - ./pve-exporter/pve.yml:/etc/prometheus/pve.yml:ro

  grafana:
    image: grafana/grafana:${GRAFANA_VERSION}
    restart: unless-stopped
    environment:
      GF_SERVER_ROOT_URL: https://grafana.exemple.fr
      GF_USERS_ALLOW_SIGN_UP: "false"
      GF_ANALYTICS_REPORTING_ENABLED: "false"
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
      - grafana_data:/var/lib/grafana
    ports:
      - "127.0.0.1:3000:3000"

volumes:
  prometheus_data:
  influxdb_data:
  grafana_data:

docker compose up -d influxdb

# organisation, bucket de 400 jours et compte administrateur
docker compose exec influxdb influx setup \
  --org captain --bucket proxmox --retention 400d \
  --username admin --password 'a-changer-tout-de-suite' --force

# jeton d'ecriture limite au seul bucket proxmox, pour les noeuds
docker compose exec influxdb influx bucket list
docker compose exec influxdb influx auth create \
  --org captain --write-bucket ID_DU_BUCKET \
  --description "ecriture proxmox"

# jeton de lecture pour Grafana
docker compose exec influxdb influx auth create \
  --org captain --read-bucket ID_DU_BUCKET \
  --description "lecture grafana"

# correspondance InfluxQL, pour les tableaux de bord
docker compose exec influxdb influx v1 dbrp create \
  --org captain --db proxmox --rp autogen \
  --bucket-id ID_DU_BUCKET --default

# commande du conteneur, a placer dans le service influxdb
influxdb3 serve --node-id supervision01 \
  --object-store file --data-dir /var/lib/influxdb3

# jeton administrateur, affiche une seule fois : le conserver au coffre
docker compose exec influxdb influxdb3 create token --admin

# la base, equivalent du bucket
docker compose exec influxdb influxdb3 create database proxmox \
  --token "$INFLUXDB3_ADMIN_TOKEN"

pvesh create /cluster/metrics/server/supervision \
  --type influxdb \
  --server 10.10.0.50 --port 8086 \
  --influxdbproto http \
  --organization captain --bucket proxmox \
  --token 'JETON_D_ECRITURE'

# la configuration est partagee par tout le cluster
cat /etc/pve/status.cfg

docker compose exec influxdb influx query --org captain \
  'from(bucket:"proxmox") |> range(start:-5m) |> group(columns:["_measurement"]) |> count()'

apt install prometheus-node-exporter

# n'ecouter que sur le reseau d'administration
echo 'ARGS="--web.listen-address=10.10.0.11:9100"' \
  > /etc/default/prometheus-node-exporter
systemctl restart prometheus-node-exporter

curl -s http://10.10.0.11:9100/metrics | head

ceph mgr module enable prometheus
# le point de collecte suit le gestionnaire actif, port 9283
ceph mgr services

pveum user add supervision@pve --comment "lecture seule pour la supervision"
pveum acl modify / --users supervision@pve --roles PVEAuditor
pveum user token add supervision@pve prometheus --privsep 0

default:
  user: supervision@pve
  token_name: prometheus
  token_value: "secret affiche a la creation du jeton"
  verify_ssl: true

global:
  scrape_interval: 30s
  evaluation_interval: 30s

alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

rule_files:
  - /etc/prometheus/regles/*.yml

scrape_configs:
  - job_name: noeuds
    static_configs:
      - targets: ["10.10.0.11:9100", "10.10.0.12:9100", "10.10.0.13:9100"]
        labels: {role: proxmox}
      - targets: ["10.10.0.21:9100", "10.10.0.22:9100"]
        labels: {role: web}

  - job_name: ceph
    honor_labels: true
    static_configs:
      - targets: ["10.10.0.11:9283", "10.10.0.12:9283", "10.10.0.13:9283"]

  - job_name: pve
    metrics_path: /pve
    params:
      module: [default]
      cluster: ["1"]
      node: ["1"]
    static_configs:
      - targets: ["10.10.0.11"]
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: pve-exporter:9221

  - job_name: sites
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets: ["https://www.exemple.fr", "https://boutique.exemple.fr"]
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115

apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus
    url: http://prometheus:9090
    isDefault: true
    jsonData:
      timeInterval: 30s

  - name: Proxmox
    type: influxdb
    uid: proxmox
    url: http://influxdb:8086
    jsonData:
      dbName: proxmox
      httpMode: POST
      httpHeaderName1: Authorization
    secureJsonData:
      httpHeaderValue1: "Token JETON_DE_LECTURE"

groups:
  - name: base
    rules:
      # une cible muette n'est pas une cible en bonne sante
      - alert: CibleInjoignable
        expr: up == 0
        for: 5m
        labels: {severity: critique}
        annotations:
          summary: "{{ $labels.instance }} ne repond plus depuis 5 minutes"

      # prevoir plutot que constater : plein dans moins de 4 heures
      - alert: DisquePleinSous4h
        expr: |
          predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 4*3600) < 0
          and node_filesystem_avail_bytes / node_filesystem_size_bytes < 0.2
        for: 30m
        labels: {severity: critique}
        annotations:
          summary: "{{ $labels.mountpoint }} sur {{ $labels.instance }} sera plein sous 4 h"

      - alert: CephDegrade
        expr: ceph_health_status > 0
        for: 10m
        labels: {severity: alerte}
        annotations:
          summary: "Ceph n'est plus en HEALTH_OK depuis 10 minutes"

      - alert: SiteEnPanne
        expr: probe_success == 0
        for: 2m
        labels: {severity: critique}
        annotations:
          summary: "{{ $labels.instance }} ne repond plus"

      - alert: CertificatBientotExpire
        expr: probe_ssl_earliest_cert_expiry - time() < 14 * 86400
        for: 1h
        labels: {severity: alerte}
        annotations:
          summary: "Le certificat de {{ $labels.instance }} expire dans moins de 14 jours"

route:
  receiver: equipe
  group_by: [alertname, instance]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - matchers: [severity="critique"]
      receiver: astreinte
      repeat_interval: 1h

receivers:
  - name: equipe
    webhook_configs:
      - url: https://chat.exemple.fr/hooks/supervision
  - name: astreinte
    webhook_configs:
      - url: https://chat.exemple.fr/hooks/astreinte
      - url: https://ntfy.exemple.net/astreinte

docker compose exec prometheus promtool check config /etc/prometheus/prometheus.yml
docker compose exec prometheus promtool check rules /etc/prometheus/regles/base.yml
curl -X POST http://127.0.0.1:9090/-/reload

# nombre de series actives
curl -s 'http://127.0.0.1:9090/api/v1/query?query=prometheus_tsdb_head_series'

# taille sur disque
docker compose exec prometheus du -sh /prometheus