Parametrise MaxHealthCheckRetries for containerd

Signed-off-by: Antonio Murdaca <runcom@redhat.com>

Author: runcom

cmd/dockerd/daemon_unix.go

@@ -62,6 +62,7 @@ func (cli *DaemonCli) getPlatformRemoteOptions() []libcontainerd.RemoteOption {
 	opts := []libcontainerd.RemoteOption{
 		libcontainerd.WithDebugLog(cli.Config.Debug),
 		libcontainerd.WithOOMScore(cli.Config.OOMScoreAdjust),
+		libcontainerd.WithMaxHealthCheckRetries(cli.Config.MaxHealthCheckRetries),
 	}
 	if cli.Config.ContainerdAddr != "" {
 		opts = append(opts, libcontainerd.WithRemoteAddr(cli.Config.ContainerdAddr))

daemon/config_unix.go

@@ -26,18 +26,19 @@ type Config struct {
 	CommonUnixConfig
 
 	// Fields below here are platform specific.
-	CgroupParent         string                   `json:"cgroup-parent,omitempty"`
-	EnableSelinuxSupport bool                     `json:"selinux-enabled,omitempty"`
-	RemappedRoot         string                   `json:"userns-remap,omitempty"`
-	Ulimits              map[string]*units.Ulimit `json:"default-ulimits,omitempty"`
-	CPURealtimePeriod    int64                    `json:"cpu-rt-period,omitempty"`
-	CPURealtimeRuntime   int64                    `json:"cpu-rt-runtime,omitempty"`
-	OOMScoreAdjust       int                      `json:"oom-score-adjust,omitempty"`
-	Init                 bool                     `json:"init,omitempty"`
-	InitPath             string                   `json:"init-path,omitempty"`
-	SeccompProfile       string                   `json:"seccomp-profile,omitempty"`
-	SigCheck             bool                     `json:"signature-verification"`
-	EnableSecrets        bool                     `json:"enable-secrets"`
+	CgroupParent          string                   `json:"cgroup-parent,omitempty"`
+	EnableSelinuxSupport  bool                     `json:"selinux-enabled,omitempty"`
+	RemappedRoot          string                   `json:"userns-remap,omitempty"`
+	Ulimits               map[string]*units.Ulimit `json:"default-ulimits,omitempty"`
+	CPURealtimePeriod     int64                    `json:"cpu-rt-period,omitempty"`
+	CPURealtimeRuntime    int64                    `json:"cpu-rt-runtime,omitempty"`
+	OOMScoreAdjust        int                      `json:"oom-score-adjust,omitempty"`
+	MaxHealthCheckRetries int                      `json:"max-health-check-retries,omitempty"`
+	Init                  bool                     `json:"init,omitempty"`
+	InitPath              string                   `json:"init-path,omitempty"`
+	SeccompProfile        string                   `json:"seccomp-profile,omitempty"`
+	SigCheck              bool                     `json:"signature-verification"`
+	EnableSecrets         bool                     `json:"enable-secrets"`
 }
 
 // bridgeConfig stores all the bridge driver specific
@@ -86,6 +87,7 @@ func (config *Config) InstallFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&config.ContainerdAddr, "containerd", "", "Path to containerd socket")
 	flags.BoolVar(&config.LiveRestoreEnabled, "live-restore", false, "Enable live restore of docker when containers are still running")
 	flags.IntVar(&config.OOMScoreAdjust, "oom-score-adjust", -500, "Set the oom_score_adj for the daemon")
+	flags.IntVar(&config.MaxHealthCheckRetries, "max-health-check-retries", 3, "Set the maximum number of health check retries before forced containerd restart")
 	flags.BoolVar(&config.Init, "init", false, "Run an init in the container to forward signals and reap processes")
 	flags.StringVar(&config.InitPath, "init-path", "", "Path to the docker-init binary")
 	flags.Int64Var(&config.CPURealtimePeriod, "cpu-rt-period", 0, "Limit the CPU real-time period in microseconds")

libcontainerd/remote_unix.go

@@ -33,7 +33,6 @@ import (
 )
 
 const (
-	maxConnectionRetryCount      = 3
 	containerdHealthCheckTimeout = 3 * time.Second
 	containerdShutdownTimeout    = 15 * time.Second
 	containerdBinary             = "docker-containerd"
@@ -45,22 +44,23 @@ const (
 
 type remote struct {
 	sync.RWMutex
-	apiClient            containerd.APIClient
-	daemonPid            int
-	stateDir             string
-	rpcAddr              string
-	startDaemon          bool
-	closeManually        bool
-	debugLog             bool
-	rpcConn              *grpc.ClientConn
-	clients              []*client
-	eventTsPath          string
-	runtime              string
-	runtimeArgs          []string
-	daemonWaitCh         chan struct{}
-	liveRestore          bool
-	oomScore             int
-	restoreFromTimestamp *timestamp.Timestamp
+	apiClient             containerd.APIClient
+	daemonPid             int
+	stateDir              string
+	rpcAddr               string
+	startDaemon           bool
+	closeManually         bool
+	debugLog              bool
+	rpcConn               *grpc.ClientConn
+	clients               []*client
+	eventTsPath           string
+	runtime               string
+	runtimeArgs           []string
+	daemonWaitCh          chan struct{}
+	liveRestore           bool
+	oomScore              int
+	maxHealthCheckRetries int
+	restoreFromTimestamp  *timestamp.Timestamp
 }
 
 // New creates a fresh instance of libcontainerd remote.
@@ -139,6 +139,8 @@ func (r *remote) UpdateOptions(options ...RemoteOption) error {
 func (r *remote) handleConnectionChange() {
 	var transientFailureCount = 0
 
+	logrus.Debugf("libcontainerd: maximum number of retries for containerd health check is %d", r.maxHealthCheckRetries)
+
 	ticker := time.NewTicker(500 * time.Millisecond)
 	defer ticker.Stop()
 	healthClient := grpc_health_v1.NewHealthClient(r.rpcConn)
@@ -162,7 +164,7 @@ func (r *remote) handleConnectionChange() {
 			// all other errors are transient
 			// Reset state to be notified of next failure
 			transientFailureCount++
-			if transientFailureCount >= maxConnectionRetryCount {
+			if transientFailureCount >= r.maxHealthCheckRetries {
 				transientFailureCount = 0
 				if utils.IsProcessAlive(r.daemonPid) {
 					logrus.Infof("killing and restarting containerd")
@@ -553,3 +555,19 @@ func (o oomScore) Apply(r Remote) error {
 	}
 	return fmt.Errorf("WithOOMScore option not supported for this remote")
 }
+
+// WithMaxHealthCheckRetries defines the maximum number of consecutive failed 'HealthCheckRequest'
+// before handleConnectionChange() forcibly kills and restarts containerd.
+func WithMaxHealthCheckRetries(cnt int) RemoteOption {
+	return retryCnt(cnt)
+}
+
+type retryCnt int
+
+func (cnt retryCnt) Apply(r Remote) error {
+	if remote, ok := r.(*remote); ok {
+		remote.maxHealthCheckRetries = int(cnt)
+		return nil
+	}
+	return fmt.Errorf("WithMaxHealthCheckRetries option not supported for this remote")
+}