etcd Monitoring

bash-5.0# cat etcd.yml

groups:

  - name: etcd

    rules:

      - alert: EtcdBackupJobFailed

        expr: kube_job_status_failed{job_name=~"etcd-backup.*"} offset 5m > 0

        for: 0m

        labels:

          severity: high

          type: etcd

        annotations:

          summary: Etcd Backup Job failed for cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Etcd Backup Job {{$labels.namespace}}/{{$labels.job_name}} failed to complete reason: {{$labels.reason}}"

      - alert: EtcdDown

        expr: up{job="etcd"} offset 5m == 0

        for: 10m

        labels:

          severity: critical

          type: etcd

        annotations:

          description: Etcd container down on cluster {{ $labels.cluster }}

          summary: "Cluster {{ $labels.cluster }}: Etcd container down on host {{ $labels.host }}"

      - alert: EtcdInsufficientMembers

        expr: count(etcd_server_id) by (cluster) % 2 == 0

        for: 1m

        labels:

          severity: critical

          type: etcd

        annotations:

          summary: Etcd insufficient members on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Etcd cluster should have an odd number of members\n  VALUE = {{ $value }}"

      - alert: EtcdNoLeader

        expr: etcd_server_has_leader == 0

        for: 1m

        labels:

          severity: critical

          type: etcd

        annotations:

          summary: Etcd no Leader on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Etcd cluster have no leader\n  VALUE = {{ $value }}"

      - alert: EtcdHighNumberOfLeaderChanges

        expr: increase(etcd_server_leader_changes_seen_total[10m] offset 5m) > 2

        for: 0m

        labels:

          severity: high

          type: etcd

        annotations:

          summary: Etcd high number of leader changes on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Etcd leader changed more than 2 times during 10 minutes\n  VALUE = {{ $value }}"

      - alert: EtcdHighNumberOfFailedGrpcRequests

        expr: sum(rate(grpc_server_handled_total{grpc_code!="OK"}[1m] offset 5m )) BY (grpc_service, grpc_method, cluster, host) / sum(rate(grpc_server_handled_total[1m] offset 5m )) BY (grpc_service, grpc_method, cluster, host) > 0.01

        for: 2m

        labels:

          severity: warning

          type: etcd

        annotations:

          summary: Etcd high number of failed GRPC requests on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: More than 1% GRPC request failure detected on Etcd host {{ $labels.host }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

      - alert: EtcdHighNumberOfFailedProposals

        expr: increase(etcd_server_proposals_failed_total[1h]) > 5

        for: 2m

        labels:

          severity: warning

          type: etcd

        annotations:

          summary: Etcd high number of failed proposals on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Etcd server got more than 5 failed proposals past hour\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

      - alert: EtcdHighFsyncDurations

        expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m] offset 5m )) > 0.5

        for: 2m

        labels:

          severity: warning

          type: etcd

        annotations:

          summary: Etcd high fsync durations on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Etcd WAL fsync duration increasing, 99th percentile is over 0.5s on Etcd host {{ $labels.host }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

      - alert: EtcdHighCommitDurations

        expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[1m] offset 5m )) > 0.25

        for: 2m

        labels:

          severity: warning

          type: etcd

        annotations:

          summary: Etcd high commit durations on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Etcd commit duration increasing, 99th percentile is over 0.25s on Etcd host {{ $labels.host }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

bash-5.0#