SCC bootstrapping must not return until it has created all defaults

Author: deads2k

hack/local-up-master/lib.sh

@@ -220,6 +220,11 @@ function localup::start_kubeapiserver() {
     for filename in ${OS_ROOT}/hack/local-up-master/kube-apiserver-manifests/*.yaml; do
        oc --config=${LOCALUP_CONFIG}/kube-apiserver/admin.kubeconfig apply -f ${filename}
     done
+
+    NON_LOOPBACK_IPV4=$(ip -o -4 addr show up primary scope global | awk '{print $4}' | cut -f1 -d'/' | head -n1)
+    for filename in ${OS_ROOT}/hack/local-up-master/openshift-apiserver-manifests/*.yaml; do
+        sed "s/NON_LOOPBACK_HOST/${NON_LOOPBACK_IPV4}/g" ${filename} | oc --config=${LOCALUP_CONFIG}/kube-apiserver/admin.kubeconfig apply -f -
+    done
 }
 
 function localup::start_kubecontrollermanager() {
@@ -264,10 +269,6 @@ function localup::start_openshiftapiserver() {
     kube::util::wait_for_url "https://${API_HOST_IP}:8444/healthz" "openshift-apiserver: " 1 ${WAIT_FOR_URL_API_SERVER} ${MAX_TIME_FOR_URL_API_SERVER} \
         || { os::log::error "check kube-apiserver logs: ${OPENSHIFT_APISERVER_LOG}" ; exit 1 ; }
 
-    NON_LOOPBACK_IPV4=$(ip -o -4 addr show up primary scope global | awk '{print $4}' | cut -f1 -d'/' | head -n1)
-    for filename in ${OS_ROOT}/hack/local-up-master/openshift-apiserver-manifests/*.yaml; do
-        sed "s/NON_LOOPBACK_HOST/${NON_LOOPBACK_IPV4}/g" ${filename} | oc --config=${LOCALUP_CONFIG}/openshift-apiserver/openshift-apiserver.kubeconfig apply -f -
-    done
 }
 
 function localup::start_openshiftcontrollermanager() {

pkg/cmd/openshift-apiserver/openshiftapiserver/openshift_apiserver.go

@@ -580,42 +580,48 @@ func (c *completedConfig) bootstrapSCC(context genericapiserver.PostStartHookCon
 	ns := bootstrappolicy.DefaultOpenShiftInfraNamespace
 	bootstrapSCCGroups, bootstrapSCCUsers := bootstrappolicy.GetBoostrapSCCAccess(ns)
 
-	// ClusterResourceQuota is served using CRD resource any status update must use JSON
+	// SCC is served using CRD resource any status update must use JSON
 	jsonLoopbackClientConfig := rest.CopyConfig(c.ExtraConfig.KubeAPIServerClientConfig)
 	jsonLoopbackClientConfig.ContentConfig.AcceptContentTypes = "application/json"
 	jsonLoopbackClientConfig.ContentConfig.ContentType = "application/json"
-
-	var securityClient securityv1client.SecurityV1Interface
-	err := wait.Poll(1*time.Second, 30*time.Second, func() (bool, error) {
-		var err error
-		securityClient, err = securityv1client.NewForConfig(jsonLoopbackClientConfig)
-		if err != nil {
-			utilruntime.HandleError(fmt.Errorf("unable to initialize client: %v", err))
-			return false, nil
-		}
-		return true, nil
-	})
+	securityClient, err := securityv1client.NewForConfig(jsonLoopbackClientConfig)
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("error getting client: %v", err))
 		return err
 	}
 
-	for _, scc := range bootstrappolicy.GetBootstrapSecurityContextConstraints(bootstrapSCCGroups, bootstrapSCCUsers) {
-		_, err := securityClient.SecurityContextConstraints().Create(scc)
-		if kapierror.IsAlreadyExists(err) {
+	// all SCC must exist before we report success
+	err = wait.PollUntil(1*time.Second, func() (bool, error) {
+		anySCCMissing := false
+		for _, scc := range bootstrappolicy.GetBootstrapSecurityContextConstraints(bootstrapSCCGroups, bootstrapSCCUsers) {
+			_, err := securityClient.SecurityContextConstraints().Create(scc)
+			if err == nil {
+				klog.Infof("Created default security context constraint %s", scc.Name)
+				continue
+			}
+			if kapierror.IsAlreadyExists(err) {
+				klog.V(4).Infof("default security context constraint %s, already exists", scc.Name)
+				continue
+			}
+			anySCCMissing = true
+			utilruntime.HandleError(fmt.Errorf("unable to create default security context constraint %s; %v", scc.Name, err))
 			continue
 		}
-		if err != nil {
-			utilruntime.HandleError(fmt.Errorf("unable to create default security context constraint %s.  Got error: %v", scc.Name, err))
-			continue
+		if anySCCMissing {
+			return false, nil
 		}
-		klog.Infof("Created default security context constraint %s", scc.Name)
+
+		return true, nil
+	}, context.StopCh)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("error creating SCC: %v", err))
+		return err
 	}
 
 	// until we only use the CRD, this has to be done twice.  Once for CRD creation, once when aggregated APIs take over.  Remove after we
 	// switch
 	go func() {
-		wait.PollUntil(10*time.Second, func() (bool, error) {
+		wait.PollUntil(5*time.Second, func() (bool, error) {
 			for _, scc := range bootstrappolicy.GetBootstrapSecurityContextConstraints(bootstrapSCCGroups, bootstrapSCCUsers) {
 				_, err := securityClient.SecurityContextConstraints().Create(scc)
 				if kapierror.IsAlreadyExists(err) {

test/util/server/server.go

@@ -515,11 +515,8 @@ func startOpenShiftAPIServer(masterConfig *configapi.MasterConfig, clientConfig
 	if err != nil {
 		return err
 	}
-	if err := waitForServerHealthy(openshiftAddr); err != nil {
-		return fmt.Errorf("Waiting for OpenShift API /healthz failed with: %v", err)
-	}
-
 	targetPort := intstr.Parse(openshiftAddr.Port())
+
 	kubeClient, err := kubernetes.NewForConfig(clientConfig)
 	if err != nil {
 		return err
@@ -598,6 +595,10 @@ func startOpenShiftAPIServer(masterConfig *configapi.MasterConfig, clientConfig
 		}
 	}
 
+	if err := waitForServerHealthy(openshiftAddr); err != nil {
+		return fmt.Errorf("Waiting for OpenShift API /healthz failed with: %v", err)
+	}
+
 	err = wait.Poll(time.Second, 3*time.Minute, func() (bool, error) {
 		discoveryClient, err := discovery.NewDiscoveryClientForConfig(clientConfig)
 		if err != nil {