2.0.1 fixes (#12)

* Major rework of networking, to support multi deployment setups

HarborManagerContainer.groovy
 * Now also modifies the supplied docker compose file to move all containers to a custom network
Container.groovy
 * Some renaming of variables and methods needed, because the groovy autogenerated methods led to confusion and inconsistency.

* HarborManagerContainer.groovy
 * Bug in bash setup script

Container.groovy
 * Bug in status()

HarborDeployment.groovy
 * Improved stopAndRemoveDeployment() so it can stop deployment even if manager container is missing

pom.xml
 * Fixed problems with shading

* Untested

DockerClientDS.groovy
 * Extendes the default DockerClientImpl with an exec command that allows a ExecConfig to be supplied

Container.groovy
 * Now uses DockerClientDS
 * runCommandInContainer now alloes user, group and working dir to be supplied

 pom.xml
  * Excluded most/all of groovy from standalone jar to not conflict with the hosting groovy version, consider perhaps instead setting groovy with scope provided

Author: farthinder

Environments/Terraform/main.tf

@@ -288,9 +288,9 @@ resource "aws_lb" "load-balancer" {
 
   name               = "${var.tags.useCase}-${var.tags.owner}-lb"
   internal           = false
-  load_balancer_type = "network"
+  load_balancer_type = "application"
   subnets            = [aws_subnet.base-stack-public-subnet.id]
-
+  enable_deletion_protection = false
 
 }
 

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.eficode</groupId>
     <artifactId>devstack</artifactId>
-    <version>2.0.0-SNAPSHOT-groovy-${groovy.major.version}</version>
+    <version>2.0.1-SNAPSHOT-groovy-${groovy.major.version}</version>
     <packaging>jar</packaging>
 
     <name>DevStack - Parent pom</name>
@@ -157,22 +157,36 @@
                             <!-- final name of the shaded jar will be ${project.artifactId}-standalone -->
                             <shadedClassifierName>standalone</shadedClassifierName>
 
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+
                             <artifactSet>
                                 <excludes>
-                                    <exclude>org.codehaus.groovy:groovy</exclude>
+                                    <exclude>org.codehaus.groovy:*</exclude>
+                                    <!--exclude>org.codehaus.groovy:groovy</exclude>
+                                    <exclude>org.codehaus.groovy:groovy-astbuilder</exclude-->
+
                                     <exclude>com.google.code.gson:gson</exclude>
                                     <exclude>org.apache.httpcomponents</exclude>
                                     <exclude>commons-*</exclude>
 
                                 </excludes>
 
                             </artifactSet>
-                            <!--relocations>
+                            <relocations>
                                 <relocation>
-                                    <pattern>kong.unirest.</pattern>
-                                    <shadedPattern>com.eficode.shaded.kong.unirest.</shadedPattern>
+                                    <pattern>com.eficode.atlassian</pattern>
+                                    <shadedPattern>com.eficode.shaded.atlassian</shadedPattern>
                                 </relocation>
-                            </relocations-->
+                            </relocations>
 
                             <!-- NOTE: Any dependencies of the project will not show up in the standalone pom.
                             This means that if those dependencies are not properly relocated and there is a class-loading conflict,

src/main/groovy/com/eficode/devstack/container/Container.groovy

@@ -1,6 +1,6 @@
 package com.eficode.devstack.container
 
-import de.gesellix.docker.client.DockerClientImpl
+import com.eficode.devstack.util.DockerClientDS
 import de.gesellix.docker.client.EngineResponseContent
 import de.gesellix.docker.client.network.ManageNetworkClient
 import de.gesellix.docker.engine.DockerClientConfig
@@ -11,6 +11,7 @@ import de.gesellix.docker.remote.api.ContainerInspectResponse
 import de.gesellix.docker.remote.api.ContainerState
 import de.gesellix.docker.remote.api.ContainerSummary
 import de.gesellix.docker.remote.api.EndpointSettings
+import de.gesellix.docker.remote.api.ExecConfig
 import de.gesellix.docker.remote.api.HostConfig
 import de.gesellix.docker.remote.api.IdResponse
 import de.gesellix.docker.remote.api.Mount
@@ -40,21 +41,20 @@ import java.util.regex.Pattern
 trait Container {
 
     Logger log = LoggerFactory.getLogger(self.class)
-    DockerClientImpl dockerClient = new DockerClientImpl()
+    DockerClientDS dockerClient = new DockerClientDS()
     ManageNetworkClient networkClient = dockerClient.getManageNetwork() as ManageNetworkClient
     abstract String containerName
     abstract String containerMainPort
     abstract String containerImage
     abstract String containerImageTag
+    ArrayList<String> containerDefaultNetworks = ["bridge"]
     ArrayList<String> customEnvVar = []
-    //String containerNetworkName = "bridge"
     String defaultShell = "/bin/bash"
     String containerId
     ArrayList<Mount> mounts = []
 
 
     void prepareBindMount(String sourceAbs, String target, boolean readOnly = true) {
-        assert !isCreated(): "Bind mounts cant be prepared for already created container"
 
         Mount newMount = new Mount().tap { m ->
             m.source = sourceAbs
@@ -89,7 +89,6 @@ trait Container {
             c.hostname = self.containerName
             c.env = self.customEnvVar
 
-
         }
 
         return containerCreateRequest
@@ -98,11 +97,11 @@ trait Container {
 
     /**
      * Create container and override default docker cmd and entrypoint
-     * @param cmd:
+     * @param cmd :
      * @param entrypoint ex: ["tail", "-f", "/dev/null"]
      * @return container id
      */
-    String createContainer(ArrayList<String> cmd, ArrayList<String> entrypoint) {
+    String createContainer(ArrayList<String> cmd = [], ArrayList<String> entrypoint = []) {
 
         assert ping(): "Error connecting to docker engine"
 
@@ -119,14 +118,14 @@ trait Container {
         EngineResponseContent response = dockerClient.createContainer(containerCreateRequest, self.containerName)
         assert response.content.warnings.isEmpty(): "Error when creating ${self.containerName} container:" + response.content.warnings.join(",")
 
+        ArrayList<Network> networks = containerDefaultNetworks.collect { createBridgeNetwork(it) }
+        assert setContainerNetworks(networks): "Error setting container networks to:" + containerDefaultNetworks
+
         containerId = response.content.id
         return containerId
 
     }
 
-    String createContainer() {
-        return createContainer([], [])
-    }
 
     boolean runOnFirstStartup() {
         return true
@@ -145,7 +144,7 @@ trait Container {
         dockerEnv.setCertPath(certPath)
         dockerEnv.setTlsVerify("1")
         dockerConfig.apply(dockerEnv)
-        dockerClient = new DockerClientImpl(dockerConfig)
+        dockerClient = new DockerClientDS(dockerConfig)
         networkClient = dockerClient.getManageNetwork() as ManageNetworkClient
 
 
@@ -248,10 +247,8 @@ trait Container {
 
         if (status == ContainerState.Status.Created) {
             return true //Created but not started
-        } else if (status == null) {
-            return true //Not even created
         } else {
-            return false
+            return status == null
         }
 
     }
@@ -270,7 +267,7 @@ trait Container {
     }
 
     ContainerState.Status status() {
-        return inspectContainer().state.status
+        return inspectContainer()?.state?.status
     }
 
     boolean stopAndRemoveContainer(Integer timeoutS = 5) {
@@ -469,28 +466,39 @@ trait Container {
      * @param networkNameOrId
      * @return Network if found, null if not
      */
-    Network getNetwork(String networkNameOrId) {
+    Network getDockerNetwork(String networkNameOrId) {
 
 
         Network network = networkClient.networks().content.find { it.name == networkNameOrId || it.id == networkNameOrId }
 
         return network
     }
+    /**
+     * Gets  networks based on name or id, note there might be multiple networks with the same name
+     * @param networkNameOrIds
+     * @return Networks if found, null if not
+     */
+    ArrayList<Network> getDockerNetworks(ArrayList<String> networkNameOrIds) {
+
+        ArrayList<Network> networks = networkClient.networks().content.findAll { it.name in networkNameOrIds || it.id in networkNameOrIds }
+
+        return networks
+    }
 
     boolean networkIsValid(Network network) {
-        return getNetwork(network.id) != null
+        return getDockerNetwork(network.id) != null
     }
 
     /**
      * Get the networks that this container is connected too
      * @return
      */
-    ArrayList<Network> getContainerNetworks() {
+    ArrayList<Network> getConnectedContainerNetworks() {
         Map<String, EndpointSettings> rawResponse = inspectContainer().networkSettings.networks
 
         ArrayList<Network> networks = []
         rawResponse.keySet().each { networkId ->
-            Network network = getNetwork(networkId)
+            Network network = getDockerNetwork(networkId)
 
             if (network != null) {
                 networks.add(network)
@@ -509,7 +517,7 @@ trait Container {
     ArrayList<Network> getContainerBridgeNetworks() {
 
 
-        return getContainerNetworks().findAll { it.driver == "bridge" }
+        return getConnectedContainerNetworks().findAll { it.driver == "bridge" }
 
     }
 
@@ -533,7 +541,7 @@ trait Container {
         log.trace("\tVerifying container was added to network")
 
 
-        if (containerNetworks.find { it.id == network.id } != null) {
+        if (connectedContainerNetworks.find { it.id == network.id } != null) {
             log.info("\tContainer was successfully added to network")
             return true
         }
@@ -550,7 +558,7 @@ trait Container {
         networkClient.disconnectNetwork(network.id, containerId)
         log.trace("\tVerifying container was disconnected from network")
 
-        if (!containerNetworks.find { it.id == network.id }) {
+        if (!connectedContainerNetworks.find { it.id == network.id }) {
             log.info("\tContainer was successfully disconnected from network")
             return true
         }
@@ -569,8 +577,7 @@ trait Container {
 
         log.info("Setting container networks")
         log.info("\tBeginning by disconnecting any networks it should no longer be connected to")
-        ArrayList networks = containerNetworks
-        containerNetworks.each { connectedNetwork ->
+        connectedContainerNetworks.each { connectedNetwork ->
 
             if (newNetworks.id.find { newNetworkId -> newNetworkId != connectedNetwork.id }) {
                 assert disconnectContainerFromNetwork(connectedNetwork): "Error disconnecting container (${containerName}) from network: ${connectedNetwork.name} (${connectedNetwork.id})"
@@ -580,7 +587,7 @@ trait Container {
         }
         log.info("\tFinished disconnecting container from unwanted networks, now connecting to new networks")
 
-        ArrayList<Network> connectedNetworks = containerNetworks
+        ArrayList<Network> connectedNetworks = connectedContainerNetworks
         newNetworks.each { wantedNetwork ->
 
             if (connectedNetworks.id.find { wantedNetwork.id }) {
@@ -644,23 +651,45 @@ trait Container {
     }
 
 
-    ArrayList<String> runBashCommandInContainer(String command, long timeoutS = 10) {
+    //Format is one of: `user`, `user:group`, `uid`, or `uid:gid`
+    ArrayList<String> runCommandInContainer(String containerId, ArrayList<String> commands, long timeoutS = 10, String userGroup = null, String workingDir = null) {
 
         log.info("Executing bash command in container:")
         log.info("\tContainer:" + self.containerName + " (${self.containerId})")
-        log.info("\tCommand:" + command)
+        log.info("\tCommand:" + commands)
         log.info("\tTimeout:" + timeoutS)
-        if (log.isTraceEnabled()) {
-            log.trace("\tDocker ping:" + dockerClient.ping().content as String)
-        }
-
 
         long cmdStart = System.currentTimeMillis()
+
+        ExecConfig execConfig = new ExecConfig()
+        execConfig.with { ex ->
+            ex.attachStdin = false
+            ex.attachStdout = true
+            ex.attachStderr = true
+            ex.detachKeys = null
+            ex.tty = false
+            ex.env = null
+            ex.cmd = commands
+            ex.privileged = null
+            ex.user = userGroup ?: null
+            ex.workingDir = workingDir ?: null
+        }
+
         ContainerCallback callBack = new ContainerCallback()
-        EngineResponse<IdResponse> response = dockerClient.exec(self.containerId, [self.defaultShell, "-c", command], callBack, Duration.ofSeconds(timeoutS))
+        dockerClient.exec(containerId, commands, callBack, Duration.ofSeconds(timeoutS), execConfig)
+
+        log.trace("\tCommand finished after:" + ((System.currentTimeMillis() - cmdStart) / 1000).round() + "s")
 
-        log.trace("\tCommand finished after:" + ((System.currentTimeMillis()-cmdStart)/1000).round() + "s" )
         return callBack.output
+
+    }
+
+
+    ArrayList<String> runBashCommandInContainer(String command, long timeoutS = 10, String user = null) {
+
+        return runCommandInContainer(self.containerId, [self.defaultShell, "-c", command], timeoutS, user)
+
+
     }