Merge pull request #612 from CodeNow/create-shared-single-host-main-ami

thejsj · web-flow · commit edfdab6a5e40 · 2017-04-09T13:54:02.000-07:00
Create shared single host main ami
diff --git a/ansible/delta-hosts/variables b/ansible/delta-hosts/variables
@@ -1,5 +1,4 @@
 [agreeable-egret:vars]
-agreeable_egret_port=65520
 egret_pg_host=delta-big-poppa.cnksgdqarobf.us-west-2.rds.amazonaws.com
 egret_pg_port=5432
 egret_pg_user=egret
diff --git a/ansible/docker-files/base/id_rsa b/ansible/docker-files/base/id_rsa
diff --git a/ansible/docker-files/base/known_hosts b/ansible/docker-files/base/known_hosts
diff --git a/ansible/gamma-hosts/variables b/ansible/gamma-hosts/variables
@@ -1,5 +1,4 @@
 [agreeable-egret:vars]
-agreeable_egret_port=65520
 egret_pg_host=gamma-big-poppa.cnksgdqarobf.us-west-2.rds.amazonaws.com:32659
 egret_pg_port=32659
 egret_pg_user=egret
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
@@ -29,6 +29,8 @@ docker_config: runnable
 slack_token: T029DEC10/B1RSX8LNS/qLLSYEEqkGddohOdE44eDf3j
 slack_channels: [ '#ops' ]
 
+sendgrid_key: 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
+
 loggly_token: f673760d-e0b3-4a93-a15e-2862ea074f91
 loggly_password: TebejAcutHeH_Ch0tR9ru4anaT6CRu*3
 loggly_username: ops
@@ -46,6 +48,7 @@ core_file_dir: /docker/app-cores
 
 # agreeable-egret
 agreeable_egret_hostname: egret.{{ domain }}
+agreeable_egret_port: 65520
 
 #angular
 angular_url: https://app.{{ domain }}
@@ -114,6 +117,7 @@ docker_listener_rollbar_key: 7b5924eccfff415d9fc00c051811e9d7
 
 # drake
 drake_hostname: drake.{{ domain }}
+drake_port: 80
 
 # ec2
 aws_access_key: "AKIAIWRXWZ4P3MIMY3LA"
diff --git a/ansible/group_vars/alpha-api-base.yml b/ansible/group_vars/alpha-api-base.yml
@@ -7,9 +7,6 @@ repo: git@github.com:CodeNow/api.git
 restart_policy: always
 has_shrinkwrap: true
 
-# for sendGrid
-sendgrid_key: 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
-
 dockerfile_enviroment: [
   "NPM_TOKEN {{ npm_token }}"
 ]
diff --git a/ansible/group_vars/alpha-api.yml b/ansible/group_vars/alpha-api.yml
@@ -18,7 +18,7 @@ container_envs: >
 container_run_opts: >
   -h {{ name }}
   -d
-  -P
+  -p {{ hosted_ports[0] }}
   -v /opt/ssl/docker/{{ name }}:/etc/ssl/docker:ro
   -v /opt/ssl/mongodb-client:/opt/ssl/mongodb-client:ro
   {{ container_envs }}
diff --git a/ansible/group_vars/alpha-big-poppa-http.yml b/ansible/group_vars/alpha-big-poppa-http.yml
@@ -13,7 +13,7 @@ dockerfile_enviroment: [
 
 dockerfile_pre_install_commands: [
   "apt-get update",
-  "apt-get install postgresql-client=9.3+154ubuntu1 -y",
+  "apt-get install postgresql-client=9.4+165+deb8u2 -y",
   "echo '//registry.npmjs.org/:_authToken=${NPM_TOKEN}' > .npmrc"
 ]
 
@@ -56,3 +56,5 @@ container_run_opts: >
   {{container_envs}}
 
 npm_start_command: run start-http
+
+container_run_args: npm {{ npm_start_command  }}
diff --git a/ansible/group_vars/alpha-big-poppa-worker.yml b/ansible/group_vars/alpha-big-poppa-worker.yml
@@ -12,7 +12,7 @@ dockerfile_enviroment: [
 
 dockerfile_pre_install_commands: [
   "apt-get update",
-  "apt-get install postgresql-client=9.3+154ubuntu1 -y",
+  "apt-get install postgresql-client=9.4+165+deb8u2 -y",
   "echo '//registry.npmjs.org/:_authToken=${NPM_TOKEN}' > .npmrc"
 ]
 
@@ -44,3 +44,5 @@ container_run_opts: >
   {{container_envs}}
 
 npm_start_command: run start-worker
+
+container_run_args: npm {{ npm_start_command  }}
diff --git a/ansible/group_vars/alpha-cream-http.yml b/ansible/group_vars/alpha-cream-http.yml
@@ -43,3 +43,5 @@ container_run_opts: >
   {{container_envs}}
 
 npm_start_command: run start-http
+
+container_run_args: npm {{ npm_start_command  }}
diff --git a/ansible/group_vars/alpha-cream-worker.yml b/ansible/group_vars/alpha-cream-worker.yml
@@ -40,3 +40,5 @@ container_run_opts: >
   {{container_envs}}
 
 npm_start_command: run start-worker
+
+container_run_args: npm {{ npm_start_command  }}
diff --git a/ansible/group_vars/alpha-detention.yml b/ansible/group_vars/alpha-detention.yml
@@ -12,6 +12,8 @@ redis_key: "frontend:{{ detention_hostname }}"
 is_redis_update_required: 'yes'
 
 dockerfile_post_install_commands: [
+  "wget -nv https://github.com/eSlider/sassc-binaries/raw/develop/dist/sassc -O /usr/bin/sass",
+  "chmod +x /usr/bin/sass",
   "npm run grunt"
 ]
 
diff --git a/ansible/group_vars/alpha-drake-http.yml b/ansible/group_vars/alpha-drake-http.yml
@@ -2,8 +2,8 @@ name: drake-http
 
 container_image: registry.runnable.com/runnable/{{ name }}
 container_tag: "{{ git_branch }}"
-hosted_ports: ["{{ drake_port }}"]
 repo: "git@github.com:CodeNow/drake.git"
+hosted_ports: ["{{ drake_port }}"]
 node_version: "4.4.7"
 npm_version: 2
 
@@ -18,7 +18,7 @@ container_envs: >
   -e DATADOG_PORT={{ datadog_port }}
   -e DATADOG_SECRET=I_solemnly_swear_that_I_am_up_to_no_good
   -e NODE_ENV={{ node_env }}
-  -e PORT={{ hosted_ports[0] }}
+  -e PORT={{ drake_port }}
   -e RABBITMQ_HOSTNAME={{ rabbit_host_address }}
   -e RABBITMQ_PASSWORD={{ rabbit_password }}
   -e RABBITMQ_PORT={{ rabbit_port }}
@@ -32,3 +32,5 @@ container_run_opts: >
   {{container_envs}}
 
 npm_start_command: run start-http
+
+container_run_args: npm {{ npm_start_command  }}
diff --git a/ansible/group_vars/alpha-drake-worker.yml b/ansible/group_vars/alpha-drake-worker.yml
@@ -25,3 +25,5 @@ container_run_opts: >
   {{container_envs}}
 
 npm_start_command: run start-worker
+
+container_run_args: npm {{ npm_start_command  }}
diff --git a/ansible/group_vars/alpha-pheidi.yml b/ansible/group_vars/alpha-pheidi.yml
@@ -6,9 +6,6 @@ repo: "git@github.com:CodeNow/{{ name }}.git"
 node_version: "4.2.2"
 npm_version: "2.14.7"
 
-# for sendGrid
-sendgrid_key: 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
-
 dockerfile_enviroment: [
   "NPM_TOKEN {{ npm_token }}"
 ]
diff --git a/ansible/group_vars/alpha-socket-server.yml b/ansible/group_vars/alpha-socket-server.yml
@@ -17,7 +17,7 @@ container_envs: >
 container_run_opts: >
   -h {{ name }}
   -d
-  -P
+  -p {{ hosted_ports[0] }}
   -v /opt/ssl/docker/{{ name }}:/etc/ssl/docker:ro
   -v /opt/ssl/mongodb-client:/opt/ssl/mongodb-client:ro
   {{ container_envs }}
diff --git a/ansible/roles/add_image_tags/tasks/main.yml b/ansible/roles/add_image_tags/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+- name: add additional tags
+  become: true
+  shell: docker tag "{{ container_image }}:{{ container_tag }}" "registry.runnable.com/runnable/{{ tag_name_item }}:{{ container_tag }}"
+  when: additional_tags is defined
+  with_items: "{{ additional_tags }}"
+  loop_control:
+    loop_var: tag_name_item
diff --git a/ansible/roles/builder/meta/main.yml b/ansible/roles/builder/meta/main.yml
diff --git a/ansible/roles/builder/tasks/main.yml b/ansible/roles/builder/tasks/main.yml
@@ -3,7 +3,7 @@
 #
 - name: Ensure Tag Deploy For Prod
   tags: deploy
-  when: not git_branch | match("^v([0-9]+)\.([0-9]+)\.([0-9]+)$") and node_env=="production-delta"
+  when: node_env is defined and node_env=="production-delta" and not git_branch | match("^v([0-9]+)\.([0-9]+)\.([0-9]+)$")
   fail: msg="only tag can be deployed on prod not {{ git_branch }}"
 
 - name: create build folder
@@ -40,8 +40,9 @@
 
 - name: ensure latest tag is deployed
   tags: deploy
-  fail: msg="Cannot deploy {{ git_branch }} because latest is {{latest_tag.stdout}}. Bypass with `-t i_am_deploying_an_old_tag`"
-  when: node_env=="production-delta" and latest_tag.stdout != git_branch and i_am_deploying_an_old_tag is not defined
+  fail: msg="Cannot deploy -{{ git_branch }}- because latest is -{{latest_tag.stdout}}-. Bypass with `-t i_am_deploying_an_old_tag`"
+  when: (node_env is not defined or node_env is defined and node_env=="production-delta") and
+        (latest_tag.stdout != git_branch and i_am_deploying_an_old_tag is not defined)
 
 - name: copy dockerfile to build folder
   tags: deploy
@@ -50,11 +51,38 @@
     src: "{{ dockerfile }}"
     dest: "{{ build_dir }}/{{ name }}"
 
+- name: copy secrets into build dir
+  tags: [ deploy ]
+  become: true
+  copy:
+    src=../../../secrets/docker-client/{{ file_name_item }}
+    dest={{ build_dir }}/{{ name }}
+    owner=ubuntu
+    group=ubuntu
+    mode=0700
+  with_items:
+    - id_rsa
+    - known_hosts
+  loop_control:
+    loop_var: file_name_item
+
 - name: build docker image and tag
   tags: deploy
   become: yes
   command: docker build {{ build_args | default("") }} --tag="{{ container_image }}:{{ container_tag }}" "{{ build_dir }}/{{ name }}"
 
+- name: remove secrets from build dir
+  tags: [ deploy ]
+  become: true
+  file:
+    path: "{{ build_dir }}/{{ name }}/{{ file_name_item }}"
+    state: absent
+  with_items:
+    - id_rsa
+    - known_hosts
+  loop_control:
+    loop_var: file_name_item
+
 - name: push docker image
   become: yes
   when: not do_not_push
diff --git a/ansible/roles/builder/templates/basic_node/Dockerfile b/ansible/roles/builder/templates/basic_node/Dockerfile
@@ -1,4 +1,8 @@
-FROM registry.runnable.com/runnable/{{ base_dockerfile }}:latest
+FROM node:{{ node_version }}
+
+{% if npm_version is defined %}
+RUN npm install npm@{{ npm_version }} -g
+{% endif %}
 
 {% if hosted_ports is defined %}
 # Expose port to Host
@@ -12,22 +16,24 @@ ENV {{ env }}
 {% endfor %}
 {% endif %}
 
-# setup node and npm versions
-RUN npm install -g n@2.1.0
-RUN n {{ node_version }} && npm install -g npm@{{ npm_version }}
-
+WORKDIR /{{ name }}
 {% if dockerfile_pre_install_commands is defined %}
 {% for command in dockerfile_pre_install_commands %}
 RUN {{ command }}
 {% endfor %}
 {% endif %}
 
+RUN mkdir /root/.ssh/
+ADD id_rsa /root/.ssh/id_rsa
+ADD known_hosts /root/.ssh/known_hosts
+RUN chmod 600 /root/.ssh/id_rsa
+
 # Add package.json from the current build context (`.` is the repo) second
 ADD ./repo/package.json /{{ name }}/package.json
 
 # Add shrinkwrap from the current build context (`.` is the repo) first
 # If we change a non first-level dependency, this will break cache as expected
-{% if has_shrinkwrap is defined %}
+{% if has_shrinkwrap is defined  and has_shrinkwrap %}
 ADD ./repo/npm-shrinkwrap.json /{{ name }}/npm-shrinkwrap.json
 {% endif %}
 
@@ -44,5 +50,8 @@ RUN {{ command }}
 {% endfor %}
 {% endif %}
 
+RUN rm /root/.ssh/id_rsa
+RUN rm /root/.ssh/known_hosts
+
 # Define default command.
 CMD ulimit -c unlimited && /usr/local/bin/npm {{ npm_start_command | default('start') }} 
diff --git a/ansible/roles/docker_squash/tasks/main.yml b/ansible/roles/docker_squash/tasks/main.yml
@@ -0,0 +1,22 @@
+---
+- name: Check to see if pip is already installed.
+  command: "pip --version"
+  ignore_errors: true
+  changed_when: false # read-only task
+  check_mode: no
+  register: pip_is_installed
+
+- name: Download pip.
+  get_url: url=https://bootstrap.pypa.io/get-pip.py dest=./
+  when: pip_is_installed.rc != 0
+
+- name: Install pip.
+  command: "python ./get-pip.py"
+  become: yes
+  when: pip_is_installed.rc != 0
+
+- name: Install docker-squash
+  become: true
+  pip:
+    name: docker-squash
+    executable: pip
diff --git a/ansible/roles/squash_image/meta/main.yml b/ansible/roles/squash_image/meta/main.yml
@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: docker_squash }
diff --git a/ansible/roles/squash_image/tasks/main.yml b/ansible/roles/squash_image/tasks/main.yml
@@ -0,0 +1,10 @@
+---
+- name: get image id for newly created image
+  become: true
+  shell: docker images -a | grep "{{ container_image }}" --color=never | awk '{print $3}'
+  register: unsquahed_image_id
+
+- name: squash newly created image
+  become: true
+  ignore_errors: true # Might already be squashed
+  shell: docker-squash "{{ unsquahed_image_id.stdout }}" -t "{{ container_image }}:{{ container_tag }}"
diff --git a/ansible/single-host-base.yml b/ansible/single-host-base.yml

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`[agreeable-egret:vars]`
`2`		`-agreeable_egret_port=65520`
`3`	`2`	`egret_pg_host=delta-big-poppa.cnksgdqarobf.us-west-2.rds.amazonaws.com`
`4`	`3`	`egret_pg_port=5432`
`5`	`4`	`egret_pg_user=egret`
Original file line number	Diff line number	Diff line change
`@@ -7,9 +7,6 @@ repo: git@github.com:CodeNow/api.git`
`7`	`7`	`restart_policy: always`
`8`	`8`	`has_shrinkwrap: true`
`9`	`9`
`10`		`-# for sendGrid`
`11`		`-sendgrid_key: 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`
`12`		`-`
`13`	`10`	`dockerfile_enviroment: [`
`14`	`11`	`"NPM_TOKEN {{ npm_token }}"`
`15`	`12`	`]`