Kubernetes API Monitoring

bash-5.0# cat k8s.yml

groups:

  - name: kube-apiserver

    rules:

#-----------------------------------   API server -------------------------------------

​

      - alert: KubeAPIServerDown

        expr: up{job="kube-apiserver"} offset 5m == 0

        for: 1m

        labels:

          severity: critical

          type: k8s

        annotations:

          description: Kubernetes API server down on cluster {{ $labels.cluster }}

          summary: "Cluster {{ $labels.cluster }}: Kube api server down on host {{ $labels.host }}"

​

      - alert: KubernetesApiServerErrors

        expr: sum(rate(apiserver_request_total{code=~"(4|5).."}[1m] offset 5m )) by (cluster, host) / sum(rate(apiserver_request_total[1m] offset 5m )) by (cluster, host) * 100 > 3

        for: 2m

        labels:

          severity: critical

          type: k8s

        annotations:

          summary: Kubernetes API server errors on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Kubernetes API server is experiencing high error rate on host {{ $labels.host }}"

​

      - alert: KubernetesApiClientErrors

        expr: (sum(rate(rest_client_requests_total{code=~"(4|5).."}[1m] offset 5m )) by (cluster, host) / sum(rate(rest_client_requests_total[1m] offset 5m )) by (cluster, host)) * 100 > 1

        for: 2m

        labels:

          severity: critical

          type: k8s

        annotations:

          summary: Kubernetes API client errors on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Kubernetes API client is experiencing high error rate on host {{ $labels.host }}"

​

#-----------------------------------   Scheduler -------------------------------------

      - alert: KubeSchedulerDown

        expr: up{job="kube-scheduler"} offset 5m == 0

        for: 1m

        labels:

          severity: critical

          type: k8s

        annotations:

          summary: Kubernetes Scheduler down on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Kube scheduler down on host {{ $labels.host }}"

​

​

#-----------------------------------   Controller -------------------------------------

      - alert: KubeControllerManagerDown

        expr: up{job="kube-controller"} offset 5m == 0

        for: 1m

        labels:

          severity: critical

          type: k8s

        annotations:

          summary: Kubernetes Controller down on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: Kube controller down on host {{ $labels.host }}"

#-----------------------------------   KubeProxy -------------------------------------

      - alert: KubeProxyDown

        expr: up{job="kube-proxy"} offset 5m == 0

        for: 1m

        labels:

          severity: critical

          type: k8s

        annotations:

          summary: KubeProxy down on cluster {{ $labels.cluster }}

          description: "Cluster {{ $labels.cluster }}: kube proxy down on host {{ $labels.host }}"

​

      - alert: KubeProxyRuleSyncLatency

        expr: histogram_quantile(0.99, sum by(le, cluster, host) (rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket[5m] offset 5m ))) > 60

        for: 2m

        labels:

         severity: warning

         type: k8s

        annotations:

          summary: Cluster {{ $labels.cluster }} is taking too long, on average, to apply kubernetes service rules to iptables.

          description: "Cluster {{ $labels.cluster }}: network rules synchronization slowing down, VALUE = {{ $value }} on host {{ $labels.host }}"bash-5.0#