File kured-telemetrics.patch of Package kured

Overview Repositories Revisions Requests Users Attributes Meta

File kured-telemetrics.patch of Package kured (Revision de3770f0ef7bcc6dc60a2bbf946d55e9)

Currently displaying revision de3770f0ef7bcc6dc60a2bbf946d55e9 , Show latest

--- cmd/kured/main.go
+++ cmd/kured/main.go	2019/03/13 12:06:12
@@ -8,7 +8,7 @@
 	"regexp"
 	"time"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -20,6 +20,7 @@
 	"github.com/weaveworks/kured/pkg/daemonsetlock"
 	"github.com/weaveworks/kured/pkg/delaytick"
 	"github.com/weaveworks/kured/pkg/notifications/slack"
+	"github.com/weaveworks/kured/pkg/notifications/telemetrics"
 )
 
 var (
@@ -35,6 +36,8 @@
 	rebootSentinel string
 	slackHookURL   string
 	slackUsername  string
+	telemetricsEnabled bool
+	telemetricsLevel int
 
 	// Metrics
 	rebootRequiredGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -42,6 +45,8 @@
 		Name:      "reboot_required",
 		Help:      "OS requires reboot due to software updates.",
 	}, []string{"node"})
+
+	log = logrus.New()
 )
 
 func init() {
@@ -74,6 +79,11 @@
 	rootCmd.PersistentFlags().StringVar(&slackUsername, "slack-username", "kured",
 		"slack username for reboot notfications")
 
+	rootCmd.PersistentFlags().BoolVarP(&telemetricsEnabled, "telemetrics", "", false,
+		"Enable telemetrics messages")
+	rootCmd.PersistentFlags().IntVarP(&telemetricsLevel, "telemetrics-level", "", 2,
+		"Only telemetrics messages with this or a higher severity level are submitted")
+
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
 	}
@@ -83,15 +93,15 @@
 func newCommand(name string, arg ...string) *exec.Cmd {
 	cmd := exec.Command(name, arg...)
 
-	cmd.Stdout = log.NewEntry(log.StandardLogger()).
+	cmd.Stdout = logrus.NewEntry(logrus.StandardLogger()).
 		WithField("cmd", cmd.Args[0]).
 		WithField("std", "out").
-		WriterLevel(log.InfoLevel)
+		WriterLevel(logrus.InfoLevel)
 
-	cmd.Stderr = log.NewEntry(log.StandardLogger()).
+	cmd.Stderr = logrus.NewEntry(logrus.StandardLogger()).
 		WithField("cmd", cmd.Args[0]).
 		WithField("std", "err").
-		WriterLevel(log.WarnLevel)
+		WriterLevel(logrus.WarnLevel)
 
 	return cmd
 }
@@ -186,6 +196,9 @@
 			log.Warnf("Error notifying slack: %v", err)
 		}
 	}
+	if telemetricsEnabled {
+	        telemetrics.NotifyDrain(nodeID)
+	}
 
 	drainCmd := newCommand("/usr/bin/kubectl", "drain",
 		"--ignore-daemonsets", "--delete-local-data", "--force", nodeID)
@@ -211,6 +224,9 @@
 			log.Warnf("Error notifying slack: %v", err)
 		}
 	}
+	if telemetricsEnabled {
+	        telemetrics.NotifyReboot(nodeID)
+	}
 
 	// Relies on hostPID:true and privileged:true to enter host mount space
 	rebootCmd := newCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/bin/systemctl", "reboot")
@@ -292,6 +308,13 @@
 	log.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
 	log.Infof("Reboot Sentinel: %s every %v", rebootSentinel, period)
 
+	if telemetricsEnabled {
+		hook, err := telemetrics.NewTelemetricsHook(telemetricsEnabled, telemetricsLevel)
+		if err == nil {
+			log.Hooks.Add(hook)
+		}
+	}
+
 	go rebootAsRequired(nodeID)
 	go maintainRebootRequiredMetric(nodeID)
 
--- kured-ds.yaml
+++ kured-ds.yaml	2019/03/13 11:45:31
@@ -54,3 +54,5 @@
 #            - --reboot-sentinel=/var/run/reboot-required
 #            - --slack-hook-url=https://hooks.slack.com/...
 #            - --slack-username=prod
+#            - --telemetrics
+#            - --telemetrics-level=2
--- pkg/notifications/telemetrics/telemetrics.go
+++ pkg/notifications/telemetrics/telemetrics.go	2019/03/13 12:17:41
@@ -0,0 +1,70 @@
+package telemetrics
+
+import (
+        "os"
+        "os/exec"
+	"fmt"
+	"github.com/sirupsen/logrus"
+	"strconv"
+)
+
+func notify(severity int, message string) error {
+        // Relies on hostPID:true and privileged:true to enter host mount space
+        telemCmd := exec.Command("/usr/bin/nsenter", "-m/proc/1/ns/mnt",
+        "--", "/usr/bin/telem-record-gen", "-s", strconv.Itoa(severity), "-c",
+        "works.weave/kured/reboot", "--payload", message)
+        if err := telemCmd.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error invoking telm-record-gen command: %v", err)
+		return err
+        }
+	return nil
+}
+
+func NotifyDrain(nodeID string) error {
+	return notify(1, fmt.Sprintf("Draining node %s", nodeID))
+}
+
+func NotifyReboot(nodeID string) error {
+	return notify(1, fmt.Sprintf("Rebooting node %s", nodeID))
+}
+
+type TelemetricsHook struct {
+        enabled bool
+	min_level int
+}
+
+func NewTelemetricsHook(enabled bool, min_level int) (*TelemetricsHook, error) {
+	return &TelemetricsHook{enabled, min_level}, nil
+}
+
+func (hook *TelemetricsHook) Fire(entry *logrus.Entry) error {
+	switch entry.Level {
+	case logrus.PanicLevel, logrus.FatalLevel:
+		if hook.min_level <= 4 {
+			return notify (4, entry.Message)
+		}
+	case logrus.ErrorLevel:
+		if hook.min_level <= 3 {
+			return notify (3, entry.Message)
+		}
+	case logrus.WarnLevel:
+		if hook.min_level <= 2 {
+			return notify (2, entry.Message)
+		}
+	case logrus.InfoLevel:
+		if hook.min_level <= 1 {
+			return notify (1, entry.Message)
+		}
+	case logrus.DebugLevel, logrus.TraceLevel:
+		if hook.min_level == 0 {
+			return notify (1, entry.Message)
+		}
+	default:
+		return nil
+	}
+	return nil
+}
+
+func (hook *TelemetricsHook) Levels() []logrus.Level {
+	return logrus.AllLevels
+}

Places

File kured-telemetrics.patch of Package kured (Revision de3770f0ef7bcc6dc60a2bbf946d55e9)

Places