Add option to handle watchdog alerts

Author: martialblog

README.md

@@ -208,6 +208,41 @@ $ check_prometheus alert --name "HostHighCpuLoad" --name "PrometheusTargetMissin
 OK - Alerts inactive | total=2 firing=0 pending=0 inactive=2
 ```
 
+#### Checking alerts via their labels
+
+The `--include-label` and `--exclude-label` options can be used to filter alerts:
+
+```bash
+$ check_prometheus alert --include-label severity=warning
+OK - 2 Alerts: 0 Firing - 0 Pending - 2 Inactive
+ \_[OK] [MysqlTooManyConnections] is inactive
+ \_[OK] [MysqlHighPreparedStatementsUtilization] is inactive
+```
+
+```bash
+$ check_prometheus alert --include-label namespace=production --exclude-label severity=info
+OK - 1 Alerts: 0 Firing - 0 Pending - 1 Inactive
+ \_[OK] [ApacheDown] is inactive
+```
+
+#### Checking watchdog alerts
+
+In Prometheus a "watchdog" or "dead man's switch" is an alert that is always firing to ensure alerting pipeline is working. The `-W, --watchdog` flag can be used to flip/negate the exit state of the plugin for these kind of alerts:
+
+```bash
+$ check_prometheus alert --name Watchdog -W --no-alerts-state 2
+[OK] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive
+\_ [OK] [Watchdog] is firing - value: 1.00 - {"alertname":"Watchdog","severity":"none"}
+|total=1 firing=1 pending=0 inactive=0
+```
+
+```bash
+$ check_prometheus alert --name Watchdog -W --no-alerts-state 2
+[CRITICAL] - 0 Alerts: 0 Firing - 0 Pending - 0 Inactive
+\_ [CRITICAL] No alerts retrieved
+|total=0 firing=0 pending=0 inactive=0
+```
+
 ## License
 
 Copyright (c) 2022 [NETWAYS GmbH](mailto:info@netways.de)

cmd/alert.go

@@ -22,6 +22,7 @@ type AlertConfig struct {
 	ExcludeLabels []string
 	IncludeLabels []string
 	ProblemsOnly  bool
+	FlipExitState bool
 	StateLabelKey string
 	NoAlertsState string
 }
@@ -163,7 +164,13 @@ inactive = 0`,
 
 				sc := result.NewPartialResult()
 
-				_ = sc.SetState(rl.GetStatus(cliAlertConfig.StateLabelKey))
+				rlStatus := rl.GetStatus(cliAlertConfig.StateLabelKey)
+				// If the negate flag is set we negate this state
+				if cliAlertConfig.FlipExitState {
+					rlStatus = negateStatus(rlStatus)
+				}
+
+				_ = sc.SetState(rlStatus)
 				sc.Output = rl.GetOutput()
 				overall.AddSubcheck(sc)
 			}
@@ -185,7 +192,13 @@ inactive = 0`,
 
 					sc := result.NewPartialResult()
 
-					_ = sc.SetState(rl.GetStatus(cliAlertConfig.StateLabelKey))
+					rlStatus := rl.GetStatus(cliAlertConfig.StateLabelKey)
+					// If the negate flag is set we negate this state
+					if cliAlertConfig.FlipExitState {
+						rlStatus = negateStatus(rlStatus)
+					}
+
+					_ = sc.SetState(rlStatus)
 					// Set the alert in the internal Type to generate the output
 					rl.Alert = alert
 					sc.Output = rl.GetOutput()
@@ -257,9 +270,12 @@ func init() {
 	fs.BoolVarP(&cliAlertConfig.ProblemsOnly, "problems", "P", false,
 		"Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed")
 
+	fs.BoolVarP(&cliAlertConfig.FlipExitState, "watchdog", "W", false,
+		"Flip the exit state for firing alerts. When this flag is set firing alerts will be OK and inactive alerts will be CRITICAL. This is intended for handling watchdog alerts")
+
 	fs.StringVarP(&cliAlertConfig.StateLabelKey, "label-key-state", "S", "",
 		"Use the given AlertRule label to override the exit state for firing alerts."+
-			"\nIf this flag is set the plugin looks for warning/critical/ok in the provided label key")
+			"\nIf this flag is set the plugin looks for the strings 'warning/critical/ok' in the provided label key")
 }
 
 // Function to convert state to integer.
@@ -314,3 +330,19 @@ func matchesLabel(labels model.LabelSet, labelsToMatch []string) bool {
 
 	return false
 }
+
+// negateStatus turns an OK state into critical and a warning/critical state into OK
+func negateStatus(state int) int {
+	switch state {
+	case check.OK:
+		return check.Critical
+	case check.Critical:
+		return check.OK
+	case check.Warning:
+		return check.OK
+	case check.Unknown:
+		return check.Unknown
+	default:
+		return check.Unknown
+	}
+}

cmd/alert_test.go

@@ -225,6 +225,15 @@ exit status 2
 			args:     []string{"run", "../main.go", "alert", "--name", "InactiveAlert"},
 			expected: "[OK] - 1 Alerts: 0 Firing - 0 Pending - 1 Inactive\n\\_ [OK] [InactiveAlert] is inactive\n|total=1 firing=0 pending=0 inactive=1\n\n",
 		},
+		{
+			name: "alert-watchdog",
+			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write(loadTestdata(alertTestDataSet2))
+			})),
+			args:     []string{"run", "../main.go", "alert", "--name", "InactiveAlert", "-W"},
+			expected: "[CRITICAL] - 1 Alerts: 0 Firing - 0 Pending - 1 Inactive\n\\_ [CRITICAL] [InactiveAlert] is inactive\n|total=1 firing=0 pending=0 inactive=1\n\nexit status 2\n",
+		},
 		{
 			name: "alert-recording-rule",
 			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

testdata/alertmanager/alert.rules

@@ -1,7 +1,14 @@
 groups:
 - name: Test Alerts for check_plugin
   rules:
-
+  - alert: Watchdog
+    annotations:
+      message: |
+        This is an alert meant to ensure that the entire alerting pipeline is functional.
+        This alert is always firing.
+    expr: vector(1)
+    labels:
+      severity: none
   - alert: PrometheusTargetMissing
     expr: up == 0
     for: 0m