Node Connectivity Monitoring

bash-5.0# cat resmgr.yml

groups:

  - name: Host Availability Alerts

    rules:

    # du_sidekick_last_checkin_seconds will be the time in seconds since a host checked in via sidekick

    # or -1 if the host is not reported by sidekick at all. The latter case may occur if

    # sidekick has restarted and a host hasn't connected since.

    - expr: >-2

        (

          sum by (host_id,host_name) (

            time() - sidekick_host_last_heartbeat_time{job="sidekickserver"} / 1000

          )

        )

        OR

        ignoring(host_name) (

          sum by (host_id,host_name) (

            (resmgr_host_up{job="resmgr"} == 0) - 1

          )

        )

      record: du_sidekick_last_checkin_seconds

​

​

  - name: Hosts disconnected

    rules:

    # resmgr says it's been down for at least 10m, sidekick says still reporting

    # NOTE: the `for` delay _must_ exceeed the cutoff (currently 600 seconds) + the scrape

    # period (1m) or else both this and host-down will fire off simultaneously

    - alert: host-disconnected

      expr: >-2

        sum by (du, host_id, host_name) (du_sidekick_last_checkin_seconds < 600) AND

        ON(host_id) resmgr_host_up{job="resmgr"} == 0

      for: 15m

      annotations:

        summary: host-disconnected

        description: "{{ $labels.host_name }} disconnected from control plane {{ $labels.du }} for more than 10 minutes"

        du: "{{ $labels.du }}"

        host_name: "{{ $labels.host_name }}"

​

​

    # resmgr says it's been down for at least 10m, sidekick "agrees"

    - alert: host-down

      expr: >-2

        sum by (du, host_id, host_name) (du_sidekick_last_checkin_seconds >= 600 OR du_sidekick_last_checkin_seconds == -1) AND

        ON(host_id) resmgr_host_up{job="resmgr"} == 0

      for: 10m

      annotations:

        summary: host-down

        description: "{{ $labels.host_name }} down {{ $labels.du }} for more than 10 minutes"

        du: "{{ $labels.du }}"

        host_name: "{{ $labels.host_name }}"

bash-5.0#