Improve Carvel Deployment Support. (#5921)

* Improve Carvel Deployment Support.

* Allow kapp-controller to select default service account.

* Provide for specifying allocateLoadBalancerNodePorts and loadBalancerClass on skipper and dataflow server services.

Author: Corneil du Plessis

build-carvel-package.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+function create_and_clear() {
+    rm -rf "$1"
+    mkdir -p "$1"
+}
+
+SCDIR=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")"))
+set -euxo pipefail
+pushd $SCDIR > /dev/null
+export DATAFLOW_VERSION=$(./mvnw help:evaluate -o -Dexpression=project.version -q -DforceStdout)
+export SKIPPER_VERSION=$(./mvnw help:evaluate -o -Dexpression=spring-cloud-skipper.version -pl spring-cloud-dataflow-parent -q -DforceStdout)
+
+if [ "$PACKAGE_VERSION" = "" ]; then
+  export PACKAGE_VERSION=$DATAFLOW_VERSION
+fi
+
+# you can launch a local docker registry using docker run -d -p 5000:5000 --name registry registry:2.7
+# export REPO_PREFIX="<local-machine-ip>:5000/"
+if [ "$REPO_PREFIX" = "" ]; then
+  REPO_PREFIX="docker.io/"
+fi
+
+export PACKAGE_BUNDLE_REPOSITORY="${REPO_PREFIX}springcloud/scdf-oss-package"
+export REPOSITORY_BUNDLE="${REPO_PREFIX}springcloud/scdf-oss-repo"
+
+export SKIPPER_REPOSITORY="springcloud/spring-cloud-skipper-server"
+export SERVER_REPOSITORY="springcloud/spring-cloud-dataflow-server"
+export CTR_VERSION=$DATAFLOW_VERSION
+export PACKAGE_NAME="scdf"
+export PACKAGE_BUNDLE_TEMPLATE="src/carvel/templates/bundle/package"
+export IMGPKG_LOCK_TEMPLATE="src/carvel/templates/imgpkg"
+export VENDIR_SRC_IN="src/carvel/config"
+export SERVER_VERSION="$DATAFLOW_VERSION"
+
+export PACKAGE_BUNDLE_GENERATED=/tmp/generated/packagebundle
+export IMGPKG_LOCK_GENERATED_IN=/tmp/generated/imgpkgin
+export IMGPKG_LOCK_GENERATED_OUT=/tmp/generated/imgpkgout
+create_and_clear $PACKAGE_BUNDLE_GENERATED
+create_and_clear $IMGPKG_LOCK_GENERATED_IN
+create_and_clear $IMGPKG_LOCK_GENERATED_OUT
+
+echo "bundle-path=$PACKAGE_BUNDLE_GENERATED"
+export SCDF_DIR="$SCDIR"
+
+sh "$SCDIR/.github/actions/build-package-bundle/build-package-bundle.sh"
+
+imgpkg push --bundle "$PACKAGE_BUNDLE_REPOSITORY:$PACKAGE_VERSION" --file "$PACKAGE_BUNDLE_GENERATED"
+
+export REPO_BUNDLE_TEMPLATE="src/carvel/templates/bundle/repo"
+
+export REPO_BUNDLE_RENDERED=/tmp/generated/reporendered
+export REPO_BUNDLE_GENERATED=/tmp/generated/repobundle
+create_and_clear $REPO_BUNDLE_RENDERED
+create_and_clear $REPO_BUNDLE_GENERATED
+
+sh "$SCDIR/.github/actions/build-repository-bundle/build-repository-bundle.sh"
+
+imgpkg push --bundle "$REPOSITORY_BUNDLE:$PACKAGE_VERSION" --file "$REPO_BUNDLE_GENERATED"
+
+popd

build-containers.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+./mvnw install -s .settings.xml -DskipTests -T 1C -am -pl :spring-cloud-dataflow-server,:spring-cloud-skipper-server,:spring-cloud-dataflow-composed-task-runner
+./mvnw spring-boot:build-image -s .settings.xml -DskipTests -T 1C -pl :spring-cloud-dataflow-server,:spring-cloud-skipper-server,:spring-cloud-dataflow-composed-task-runner

src/carvel/config/dataflow-svc.yml

@@ -1,6 +1,9 @@
 #@ load("@ytt:data", "data")
 #@ load("dataflow.star", "service_spec_type")
-
+#@ load("dataflow.star", "service_spec_type_loadbalancer")
+#@ load("dataflow.star", "service_spec_allocate_load_balancer_node_ports")
+#@ load("dataflow.star", "has_service_spec_load_balancer_class")
+#@ load("dataflow.star", "service_spec_load_balancer_class")
 kind: Service
 apiVersion: v1
 metadata:
@@ -18,5 +21,11 @@ spec:
   - port: 80
     targetPort: 9393
     name: scdf-server
+  #@ if service_spec_type_loadbalancer():
+  allocateLoadBalancerNodePorts: #@ service_spec_allocate_load_balancer_node_ports()
+    #@ if has_service_spec_load_balancer_class():
+  loadBalancerClass: #@ service_spec_load_balancer_class()
+    #@ end
+  #@ end
   selector:
     app: scdf-server

src/carvel/config/dataflow.star

@@ -82,11 +82,30 @@ end
 def image_pull_secrets():
   return [{"name": registry_secret_ref()}]
 end
+def has_service_spec_type():
+  return non_empty_string(data.values.scdf.server.service.type)
+end
+
+def service_spec_type_loadbalancer():
+  return non_empty_string(data.values.scdf.server.service.type) and data.values.scdf.server.service.type == 'LoadBalancer'
+end
 
 def service_spec_type():
   return data.values.scdf.server.service.type
 end
 
+def service_spec_allocate_load_balancer_node_ports():
+  return data.values.scdf.server.service.allocateLoadBalancerNodePorts
+end
+
+def has_service_spec_load_balancer_class():
+  return non_empty_string(data.values.scdf.server.service.loadBalancerClass)
+end
+
+def service_spec_load_balancer_class():
+  return data.values.scdf.server.service.loadBalancerClass
+end
+
 def context_path():
   return data.values.scdf.server.contextPath
 end

src/carvel/config/skipper-svc.yml

@@ -1,5 +1,9 @@
 #@ load("@ytt:data", "data")
 #@ load("skipper.star", "service_spec_type")
+#@ load("skipper.star", "service_spec_type_loadbalancer")
+#@ load("skipper.star", "service_spec_allocate_load_balancer_node_ports")
+#@ load("skipper.star", "has_service_spec_load_balancer_class")
+#@ load("skipper.star", "service_spec_load_balancer_class")
 
 apiVersion: v1
 kind: Service
@@ -16,5 +20,11 @@ spec:
   ports:
   - port: 80
     targetPort: 7577
+  #@ if service_spec_type_loadbalancer():
+  allocateLoadBalancerNodePorts: #@ service_spec_allocate_load_balancer_node_ports()
+    #@ if has_service_spec_load_balancer_class():
+  loadBalancerClass: #@ service_spec_load_balancer_class()
+    #@ end
+  #@ end
   selector:
     app: skipper

src/carvel/config/skipper.star

@@ -90,6 +90,23 @@ def service_spec_type():
   return data.values.scdf.skipper.service.type
 end
 
+def service_spec_type_loadbalancer():
+  return non_empty_string(data.values.scdf.skipper.service.type) and data.values.scdf.skipper.service.type == 'LoadBalancer'
+end
+
+def service_spec_allocate_load_balancer_node_ports():
+  return data.values.scdf.skipper.service.allocateLoadBalancerNodePorts
+  end
+
+def has_service_spec_load_balancer_class():
+  return non_empty_string(data.values.scdf.skipper.service.loadBalancerClass)
+  end
+
+def service_spec_load_balancer_class():
+  return data.values.scdf.skipper.service.loadBalancerClass
+end
+
 def skipper_has_password():
   return non_empty_string(data.values.scdf.skipper.database.password)
-end
+end
+

src/carvel/config/values/values.yml

@@ -15,6 +15,8 @@ scdf:
       digest: ""
     service:
       type: ClusterIP
+      allocateLoadBalancerNodePorts: true
+      loadBalancerClass: ""
     resources:
       limits:
         cpu: ""
@@ -59,6 +61,8 @@ scdf:
       digest: ""
     service:
       type: ClusterIP
+      allocateLoadBalancerNodePorts: true
+      loadBalancerClass: ""
     resources:
       limits:
         cpu: ""

src/carvel/templates/bundle/repo/values-schema.yml

@@ -79,6 +79,12 @@ components:
                         - ClusterIP
                         - ExternalName
                       description: Service type
+                    allocateLoadBalancerNodePorts:
+                      type: boolean
+                      description: Indicates if load balancer should create node ports. Default is true
+                    loadBalancerClass:
+                      type: string
+                      description: Determines a specific configured type of load balancer.
                 resources:
                   type: object
                   properties:
@@ -185,6 +191,12 @@ components:
                         - ClusterIP
                         - ExternalName
                       description: Service type
+                    allocateLoadBalancerNodePorts:
+                      type: boolean
+                      description: Indicates if load balancer should create node ports. Default is true
+                    loadBalancerClass:
+                      type: string
+                      description: Determines a specific configured type of load balancer.
                 resources:
                   type: object
                   properties:

src/carvel/test/ordering.test.ts

@@ -29,7 +29,6 @@ describe('ordering', () => {
     expect(findAnnotation(skipperDeployment, 'kapp.k14s.io/change-group')).toBe('scdf.tanzu.vmware.com/skipper');
     expect(findAnnotation(dataflowService, 'kapp.k14s.io/change-group')).toBe('scdf.tanzu.vmware.com/server');
     expect(findAnnotation(dataflowDeployment, 'kapp.k14s.io/change-group')).toBe('scdf.tanzu.vmware.com/server');
-
     expect(findAnnotations(dataflowService, 'kapp.k14s.io/change-rule')).toContainAnyValues([
       'upsert after upserting scdf.tanzu.vmware.com/skipper'
     ]);

src/carvel/test/servers.test.ts

@@ -37,19 +37,25 @@ describe('servers', () => {
       dataValueYamls: [
         ...DEFAULT_REQUIRED_DATA_VALUES,
         'scdf.server.service.type=LoadBalancer',
-        'scdf.skipper.service.type=LoadBalancer'
+        'scdf.server.service.allocateLoadBalancerNodePorts=false',
+        'scdf.skipper.service.type=ClusterIP',
+        'scdf.skipper.service.allocateLoadBalancerNodePorts=true'
       ]
     });
     expect(result.success, result.stderr).toBeTruthy();
     const yaml = result.stdout;
 
     const dataflowService = findService(yaml, SCDF_SERVER_NAME);
+    console.log(dataflowService);
     expect(dataflowService).toBeTruthy();
     expect(dataflowService?.spec?.type).toBe('LoadBalancer');
 
+    expect(dataflowService?.spec?.allocateLoadBalancerNodePorts).toBe(false);
+
     const skipperService = findService(yaml, SKIPPER_NAME);
     expect(skipperService).toBeTruthy();
-    expect(skipperService?.spec?.type).toBe('LoadBalancer');
+    expect(skipperService?.spec?.type).toBe('ClusterIP');
+    expect(skipperService?.spec?.allocateLoadBalancerNodePorts).toBeFalsy();
   });
 
   it('should have tagged images', async () => {