Calico Monitoring

bash-5.0# cat calico.yml groups: - name: calico.rules rules: #----------------------------------- Calico Node ------------------------------------- # Felix Prometheus statistics: https://projectcalico.docs.tigera.io/reference/felix/prometheus - alert: PromHTTPRequestErrors expr: (sum(rate(promhttp_metric_handler_requests_total{job="calico-node",code=~"(4|5).."}[5m] offset 5m )) by (instance, job, cluster, host) / sum(rate(promhttp_metric_handler_requests_total{job="calico-node"}[5m] offset 5m )) by (instance, job, cluster, host)) * 100 > 1 for: 10m labels: severity: warning type: calico-node annotations: description: "Cluster {{ $labels.cluster }}: HTTP requests errors on host {{ $labels.host }}." summary: Calico HTTP requests errors on cluster {{ $labels.cluster }} - alert: CalicoDatapaneFailuresHigh expr: increase(felix_int_dataplane_failures[1h] offset 5m) > 5 for: 1h labels: severity: warning type: calico-node annotations: description: 'Felix cluster {{ $labels.cluster }} has seen {{ $value }} dataplane failures within the last hour' summary: 'A high number of dataplane failures within Felix are happening' - alert: CalicoIpsetErrorsHigh expr: increase(felix_ipset_errors[1h] offset 5m) > 5 for: 1h labels: severity: warning type: calico-node annotations: description: 'Felix cluster {{ $labels.cluster }} has seen {{ $value }} ipset errors within the last hour' summary: 'A high number of ipset errors within Felix are happening' - alert: CalicoIptableSaveErrorsHigh expr: increase(felix_iptables_save_errors[1h] offset 5m) > 5 for: 1h labels: severity: warning type: calico-node annotations: description: 'Felix cluster {{ $labels.cluster }} has seen {{ $value }} iptable save errors within the last hour' summary: 'A high number of iptable save errors within Felix are happening' - alert: CalicoIptableRestoreErrorsHigh expr: increase(felix_iptables_restore_errors[1h] offset 5m) > 5 for: 1h labels: severity: warning type: calico-node annotations: description: 'Felix cluster {{ $labels.cluster }} has seen {{ $value }} iptable restore errors within the last hour' summary: 'A high number of iptable restore errors within Felix are happening' #----------------------------------- Calico Kube Controllers ----------------------- # kube-controllers Prometheus statistics: https://projectcalico.docs.tigera.io/reference/kube-controllers/prometheus # curl http://<pod_ip>:9094/metrics #----------------------------------- Calico Typha ---------------------------------- # Typha Prometheus statistics: https://projectcalico.docs.tigera.io/reference/typha/prometheus - alert: TyphaPingLatency expr: rate(typha_ping_latency_sum[1m] offset 5m ) / rate(typha_ping_latency_count[1m] offset 5m ) > 0.1 and rate(typha_ping_latency_count[1m] offset 5m ) > 0 for: 2m labels: severity: warning type: calico-typha annotations: summary: Typha Round-trip ping latency to client (cluster {{ $labels.cluster }}) description: "Typha latency is growing (ping operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - alert: TyphaClientWriteLatency expr: rate(typha_client_write_latency_secs_sum[1m] offset 5m) / rate(typha_client_write_latency_secs_count[1m] offset 5m) > 0.1 and rate(typha_client_write_latency_secs_count[1m] offset 5m ) > 0 for: 2m labels: severity: warning type: calico-typha annotations: summary: Typha unusual write latency (instance {{ $labels.cluster }}) description: "Typha client latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" bash-5.0#