sdn: move CNI config file handling to pkg/cmd/openshift-sdn

The openshift-sdn command better knows when the full network
plugin is ready than just the OVS side does.

Author: dcbw

pkg/cmd/openshift-sdn/cmd.go

@@ -15,6 +15,7 @@ import (
 
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/record"
 	kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
 	"k8s.io/kubernetes/pkg/util/interrupt"
 
@@ -28,19 +29,23 @@ import (
 	"github.com/openshift/origin/pkg/version"
 )
 
+const openshiftCNIFile string = "80-openshift-network.conf"
+
 // OpenShiftSDN stores the variables needed to initialize the real networking
 // processess from the command line.
 type OpenShiftSDN struct {
 	ConfigFilePath            string
 	KubeConfigFilePath        string
 	URLOnlyKubeConfigFilePath string
+	cniConfFile               string
 
 	NodeConfig  *configapi.NodeConfig
 	ProxyConfig *kubeproxyconfig.KubeProxyConfiguration
 
-	informers *informers
-	OsdnNode  *sdnnode.OsdnNode
-	OsdnProxy *sdnproxy.OsdnProxy
+	informers   *informers
+	OsdnNode    *sdnnode.OsdnNode
+	sdnRecorder record.EventRecorder
+	OsdnProxy   *sdnproxy.OsdnProxy
 }
 
 var networkLong = `
@@ -97,6 +102,11 @@ func (sdn *OpenShiftSDN) Run(c *cobra.Command, errout io.Writer, stopCh chan str
 		klog.Fatal(err)
 	}
 
+	// Exit early if we can't create the CNI config file directory
+	if err := os.MkdirAll(filepath.Base(sdn.cniConfFile), 0755); err != nil {
+		klog.Fatal(err)
+	}
+
 	// Set up a watch on our config file; if it changes, we should exit -
 	// (we don't have the ability to dynamically reload config changes).
 	if err := watchForChanges(sdn.ConfigFilePath, stopCh); err != nil {
@@ -165,6 +175,12 @@ func (sdn *OpenShiftSDN) ValidateAndParse() error {
 		return err
 	}
 
+	cniConfDir := "/etc/cni/net.d"
+	if val, ok := sdn.NodeConfig.KubeletArguments["cni-conf-dir"]; ok && len(val) == 1 {
+		cniConfDir = val[0]
+	}
+	sdn.cniConfFile = filepath.Join(cniConfDir, openshiftCNIFile)
+
 	return nil
 }
 
@@ -204,6 +220,11 @@ func (sdn *OpenShiftSDN) Start(stopCh <-chan struct{}) error {
 	sdn.runProxy()
 	sdn.informers.start(stopCh)
 
+	klog.V(2).Infof("openshift-sdn network plugin registering startup")
+	if err := sdn.writeConfigFile(); err != nil {
+		klog.Fatal(err)
+	}
+	klog.V(2).Infof("openshift-sdn network plugin ready")
 	return nil
 }
 

pkg/cmd/openshift-sdn/sdn.go

@@ -1,6 +1,7 @@
 package openshift_sdn
 
 import (
+	"io/ioutil"
 	"strings"
 
 	kclientv1 "k8s.io/api/core/v1"
@@ -26,10 +27,6 @@ func (sdn *OpenShiftSDN) initSDN() error {
 	if val, ok := sdn.NodeConfig.KubeletArguments["cni-bin-dir"]; ok && len(val) == 1 {
 		cniBinDir = val[0]
 	}
-	cniConfDir := "/etc/cni/net.d"
-	if val, ok := sdn.NodeConfig.KubeletArguments["cni-conf-dir"]; ok && len(val) == 1 {
-		cniConfDir = val[0]
-	}
 
 	// dockershim + kube CNI driver delegates hostport handling to plugins,
 	// while CRI-O handles hostports itself. Thus we need to disable the
@@ -38,7 +35,7 @@ func (sdn *OpenShiftSDN) initSDN() error {
 
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartRecordingToSink(&kv1core.EventSinkImpl{Interface: sdn.informers.KubeClient.CoreV1().Events("")})
-	eventRecorder := eventBroadcaster.NewRecorder(scheme.Scheme, kclientv1.EventSource{Component: "openshift-sdn", Host: sdn.NodeConfig.NodeName})
+	sdn.sdnRecorder = eventBroadcaster.NewRecorder(scheme.Scheme, kclientv1.EventSource{Component: "openshift-sdn", Host: sdn.NodeConfig.NodeName})
 
 	var err error
 	sdn.OsdnNode, err = sdnnode.New(&sdnnode.OsdnNodeConfig{
@@ -47,7 +44,6 @@ func (sdn *OpenShiftSDN) initSDN() error {
 		SelfIP:             sdn.NodeConfig.NodeIP,
 		RuntimeEndpoint:    runtimeEndpoint,
 		CNIBinDir:          cniBinDir,
-		CNIConfDir:         cniConfDir,
 		MTU:                sdn.NodeConfig.NetworkConfig.MTU,
 		NetworkClient:      sdn.informers.NetworkClient,
 		KClient:            sdn.informers.KubeClient,
@@ -57,7 +53,7 @@ func (sdn *OpenShiftSDN) initSDN() error {
 		MasqueradeBit:      sdn.ProxyConfig.IPTables.MasqueradeBit,
 		ProxyMode:          sdn.ProxyConfig.Mode,
 		EnableHostports:    enableHostports,
-		Recorder:           eventRecorder,
+		Recorder:           sdn.sdnRecorder,
 	})
 	return err
 }
@@ -66,3 +62,18 @@ func (sdn *OpenShiftSDN) initSDN() error {
 func (sdn *OpenShiftSDN) runSDN() error {
 	return sdn.OsdnNode.Start()
 }
+
+func (sdn *OpenShiftSDN) writeConfigFile() error {
+	// Make an event that openshift-sdn started
+	sdn.sdnRecorder.Eventf(&kclientv1.ObjectReference{Kind: "Node", Name: sdn.NodeConfig.NodeName}, kclientv1.EventTypeNormal, "Starting", "openshift-sdn done initializing node networking.")
+
+	// Write our CNI config file out to disk to signal to kubelet that
+	// our network plugin is ready
+	return ioutil.WriteFile(sdn.cniConfFile, []byte(`
+{
+  "cniVersion": "0.3.1",
+  "name": "openshift-sdn",
+  "type": "openshift-sdn"
+}
+`), 0644)
+}

pkg/network/node/node.go

@@ -4,10 +4,7 @@ package node
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net"
-	"os"
-	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -47,10 +44,7 @@ import (
 	"github.com/openshift/origin/pkg/util/ovs"
 )
 
-const (
-	openshiftCNIFile = "80-openshift-network.conf"
-	hostLocalDataDir = "/var/lib/cni/networks"
-)
+const hostLocalDataDir = "/var/lib/cni/networks"
 
 type osdnPolicy interface {
 	Name() string
@@ -77,7 +71,6 @@ type OsdnNodeConfig struct {
 	MTU             uint32
 	EnableHostports bool
 	CNIBinDir       string
-	CNIConfDir      string
 
 	NetworkClient networkclient.Interface
 	KClient       kubernetes.Interface
@@ -105,7 +98,6 @@ type OsdnNode struct {
 	useConnTrack       bool
 	iptablesSyncPeriod time.Duration
 	mtu                uint32
-	cniDirPath         string
 
 	// Synchronizes operations on egressPolicies
 	egressPoliciesLock sync.Mutex
@@ -151,10 +143,6 @@ func New(c *OsdnNodeConfig) (*OsdnNode, error) {
 		return nil, fmt.Errorf("%q plugin is not compatible with proxy-mode %q", c.PluginName, c.ProxyMode)
 	}
 
-	// If our CNI config file exists, remove it so that kubelet doesn't think
-	// we're ready yet
-	os.Remove(filepath.Join(c.CNIConfDir, openshiftCNIFile))
-
 	if err := c.setNodeIP(); err != nil {
 		return nil, err
 	}
@@ -182,7 +170,6 @@ func New(c *OsdnNodeConfig) (*OsdnNode, error) {
 		kubeInformers:      c.KubeInformers,
 		networkInformers:   c.NetworkInformers,
 		egressIP:           newEgressIPWatcher(oc, c.SelfIP, c.MasqueradeBit),
-		cniDirPath:         c.CNIConfDir,
 
 		runtimeEndpoint: c.RuntimeEndpoint,
 		// 2 minutes is the current default value used in kubelet
@@ -337,34 +324,11 @@ func (node *OsdnNode) Start() error {
 		}
 	}
 
-	if err := os.MkdirAll(node.cniDirPath, 0755); err != nil {
-		return err
-	}
-
 	go kwait.Forever(node.policy.SyncVNIDRules, time.Hour)
 	go kwait.Forever(func() {
 		gatherPeriodicMetrics(node.oc.ovs)
 	}, time.Minute*2)
 
-	klog.V(2).Infof("openshift-sdn network plugin registering startup")
-
-	// Make an event that openshift-sdn started
-	node.recorder.Eventf(&corev1.ObjectReference{Kind: "Node", Name: node.hostName}, corev1.EventTypeNormal, "Starting", "Starting openshift-sdn.")
-
-	// Write our CNI config file out to disk to signal to kubelet that
-	// our network plugin is ready
-	err = ioutil.WriteFile(filepath.Join(node.cniDirPath, openshiftCNIFile), []byte(`
-{
-  "cniVersion": "0.3.1",
-  "name": "openshift-sdn",
-  "type": "openshift-sdn"
-}
-`), 0644)
-	if err != nil {
-		return err
-	}
-
-	klog.V(2).Infof("openshift-sdn network plugin ready")
 	return nil
 }