Merge branch 'master' into SAN-4042-combo-log-refactor

Christopher M. Neill · Christopher M. Neill · commit 1ae13365d1af · 2016-04-21T12:06:02.000-07:00
diff --git a/ansible/.gitignore b/ansible/.gitignore
@@ -1 +1,2 @@
+_cache
 *.pyc
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
@@ -1,3 +1,3 @@
 [defaults]
 # Required so `sudo: yes` does not lose the environment variables, which hold the ssh-agent socket
-sudo_flags=-HE
+sudo_flags = -HE
diff --git a/ansible/delta-hosts/variables b/ansible/delta-hosts/variables
@@ -27,8 +27,16 @@ drake_http_rollbar_token=52ad749ddb8e47b2a8e15312b6b300fb
 drake_worker_rollbar_token=14152b8572034943b714da27ca607698
 
 [eru:vars]
+eru_subdomain=eru
 eru_github_id=46a23f5f99f0aa9460f8
 eru_github_secret=a0336d72e3d540fb9fbbed2c123a81e1cb329dab
+eru_aws_access_key_id=AKIAIFCVEISSC5JMPWDA
+eru_aws_secret_access_key=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
+eru_aws_environment=delta
+eru_mongodb_database=delta
+eru_mongodb_password=tilde-rawboned-lotus-hideaway-diastole
+eru_mongodb_username=eru
+eru_mongodb_replset=delta-rs0
 
 [khronos:vars]
 intercom_api_key=14771f14efb617900724a16345e57beb55ba9beb
diff --git a/ansible/epsilon-hosts/variables b/ansible/epsilon-hosts/variables
@@ -26,8 +26,16 @@ drake_http_rollbar_token=52ad749ddb8e47b2a8e15312b6b300fb
 drake_worker_rollbar_token=14152b8572034943b714da27ca607698
 
 [eru:vars]
-eru_github_id=8abb08f83f6d1c52bd1a
-eru_github_secret=74a23ee56486d57b14f292283cb04625f600917c
+eru_subdomain=eru
+eru_github_id=1834e6be0811db20d219
+eru_github_secret=dda712ce9bdb92cd9187b14c0897319e90dd5462
+eru_aws_access_key_id=AKIAIFCVEISSC5JMPWDA
+eru_aws_secret_access_key=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
+eru_aws_environment=epsilon
+eru_mongodb_database=epsilon
+eru_mongodb_password=success-nan-europium-rerun-sheep
+eru_mongodb_username=eru
+eru_mongodb_replset=epsilon-rs0
 
 [khronos:vars]
 khronos_mongo_auth=api:3f5210b8-8fe3-11e5-8e62-07b6eff19ecb
diff --git a/ansible/eru.yml b/ansible/eru.yml
@@ -1,12 +1,18 @@
 ---
-- hosts: redis
 - hosts: consul
+- hosts: mongodb
+- hosts: redis
 
 - hosts: eru
   vars_files:
-    - "group_vars/alpha-eru.yml"
+  - group_vars/alpha-eru.yml
+  roles:
+  - { role: notify, tags: [ notify ] }
+  - { role: builder, tags: [ build ] }
+  - role: container_start
+
+- hosts: socket-server-proxy
+  vars_files:
+  - group_vars/alpha-eru.yml
   roles:
-  - { role: notify, tags: "notify" }
-  - { role: redis_key, tags: ["setup", "redis_key"] }
-  - { role: builder, tags: "build" }
-  - { role: container_kill_start }
+  - role: eru
diff --git a/ansible/gamma-hosts/variables b/ansible/gamma-hosts/variables
@@ -26,8 +26,16 @@ drake_http_rollbar_token=52ad749ddb8e47b2a8e15312b6b300fb
 drake_worker_rollbar_token=14152b8572034943b714da27ca607698
 
 [eru:vars]
+eru_subdomain=eru
 eru_github_id=8abb08f83f6d1c52bd1a
 eru_github_secret=74a23ee56486d57b14f292283cb04625f600917c
+eru_aws_access_key_id=AKIAIFCVEISSC5JMPWDA
+eru_aws_secret_access_key=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
+eru_aws_environment=gamma
+eru_mongodb_database=gamma
+eru_mongodb_password=success-nan-europium-rerun-sheep
+eru_mongodb_username=eru
+eru_mongodb_replset=gamma-rs0
 
 [khronos:vars]
 khronos_mongo_auth=api:3f5210b8-8fe3-11e5-8e62-07b6eff19ecb
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
@@ -95,8 +95,6 @@ aws_access_key: "AKIAIB3IJCCJZQWQMVSQ"
 aws_secret_key: "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"
 
 # eru
-eru_port: 57831
-eru_hostname: admin.{{ domain }}
 
 # filibuster
 filibuster_port: 3112
diff --git a/ansible/group_vars/alpha-api-base.yml b/ansible/group_vars/alpha-api-base.yml
@@ -9,7 +9,7 @@ repo: git@github.com:CodeNow/api.git
 sendgrid_key: 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
 
 # shared ENV's between api services
-api_base_container_envs: >
+api_base_container_envs: >-
   -e AWS_ACCESS_KEY_ID={{ api_aws_access_key_id }}
   -e AWS_SECRET_ACCESS_KEY={{ api_aws_secret_access_key }}
   -e COOKIE_DOMAIN={{ domain }}
diff --git a/ansible/group_vars/alpha-eru.yml b/ansible/group_vars/alpha-eru.yml
@@ -1,26 +1,40 @@
+---
 name: eru
 
 container_image: registry.runnable.com/runnable/{{ name }}
 container_tag: "{{ git_branch }}"
 repo: git@github.com:CodeNow/{{ name }}.git
-hosted_ports: ["{{ eru_port }}"]
-node_version: "0.12.0"
-npm_version: "2.1.18"
+hosted_ports: [ "5501", "5502" ]
+node_version: lts
+npm_version: 2
+has_shrinkwrap: true
 
-# for redis
-redis_key: frontend:{{ eru_hostname }}
-is_redis_update_required: yes
+dockerfile_post_install_commands:
+- apt-get update && apt-get install -y supervisor
+- npm run build
 
-container_envs: >
-  -e ADMIN_DOMAIN={{ eru_hostname }}
-  -e ADMIN_GITHUB_ID={{ eru_github_id }}
-  -e ADMIN_GITHUB_SECRET={{ eru_github_secret }}
-  -e API_URL={{ api_url }}
-  -e RUNNABLE_HOST={{ domain }}
-  -e PORT={{ hosted_ports[0] }}
+container_envs: >-
+  -e AWS_ACCESS_KEY={{ eru_aws_access_key_id }}
+  -e AWS_ENVIRONMENT={{ eru_aws_environment }}
+  -e AWS_SECRET_KEY={{ eru_aws_secret_access_key }}
+  -e DOMAIN={{ eru_subdomain }}.{{ domain }}
+  -e GITHUB_CLIENT_ID={{ eru_github_id }}
+  -e GITHUB_CLIENT_SECRET={{ eru_github_secret }}
+  -e MONGODB_DATABASE={{ eru_mongodb_database }}
+  -e MONGODB_PASSWORD={{ eru_mongodb_password }}
+  -e MONGODB_HOSTS={{ mongo_hosts }}
+  -e MONGODB_REPLSET={{ eru_mongodb_replset }}
+  -e MONGODB_USERNAME={{ eru_mongodb_username }}
+  -e NODE_ENV=production
+  -e REDIS_HOSTNAME={{ redis_host_address }}
+  -e REDIS_PORT={{ redis_port }}
+  -e RUNNABLE_DOMAIN={{ domain }}
 
 container_run_opts: >
   -h {{ name }}
   -d
-  -p {{ hosted_ports[0] }}:{{ hosted_ports[0] }}
+  -P
+  -v /var/log:/var/log:rw
   {{ container_envs }}
+
+container_run_args: supervisord --configuration supervisord.conf --nodaemon
diff --git a/ansible/group_vars/alpha-optimus.yml b/ansible/group_vars/alpha-optimus.yml
@@ -1,15 +1,15 @@
-name: "optimus"
+name: optimus
 
-container_image: "registry.runnable.com/runnable/{{ name }}"
+container_image: registry.runnable.com/runnable/{{ name }}
 container_tag: "{{ git_branch }}"
-repo: "git@github.com:CodeNow/{{ name }}.git"
+repo: git@github.com:CodeNow/{{ name }}.git
 hosted_ports: ["{{ optimus_port }}"]
 node_version: "4.3.2"
 npm_version: "2.8.3"
 
 # for redis
-redis_key: "frontend:{{ optimus_hostname }}"
-is_redis_update_required: 'yes'
+redis_key: frontend:{{ optimus_hostname }}
+is_redis_update_required: yes
 
 container_envs: >
   -e NODE_ENV={{ node_env }}
@@ -26,4 +26,4 @@ container_run_opts: >
   -h {{ name }}
   -d
   -P
-  {{container_envs}}
+  {{ container_envs }}
diff --git a/ansible/oneOffScripts/resetWeave.yml b/ansible/oneOffScripts/resetWeave.yml
@@ -0,0 +1,6 @@
+- hosts: docks
+  gather_facts: no
+  tasks:
+  - name: reset weave
+    become: true
+    shell: 'sudo weave reset'
diff --git a/ansible/roles/build_essential/tasks/main.yml b/ansible/roles/build_essential/tasks/main.yml
@@ -1,8 +1,8 @@
 ---
 - name: install build essentials
-  sudo: true
+  become: yes
   action: apt
-    pkg="build-essential"
-    state=present
-    update_cache=yes
-    cache_valid_time=604800
+    pkg: build-essential
+    state: present
+    update_cache: yes
+    cache_valid_time: 604800
diff --git a/ansible/roles/builder/defaults/main.yml b/ansible/roles/builder/defaults/main.yml
@@ -1,3 +1,3 @@
 ---
-build_dir: /opts/builds/docker_build/
+build_dir: /opts/builds/docker_build
 npm_start_command: start
diff --git a/ansible/roles/builder/tasks/main.yml b/ansible/roles/builder/tasks/main.yml
@@ -3,31 +3,33 @@
 - name: create build folder
   become: true
   file:
-    path="{{ build_dir }}/{{ name }}"
-    state=directory
+    path: "{{ build_dir }}/{{ name }}"
+    state: directory
 
 - name: pull the git repository
   tags: deploy
   become: true
   git:
-    repo="{{ repo }}"
-    dest="{{ build_dir }}/{{ name }}/repo"
-    version="{{ git_branch }}"
-    update=yes
-    accept_hostkey=True
-    force=yes
+    repo: "{{ repo }}"
+    dest: "{{ build_dir }}/{{ name }}/repo"
+    version: "{{ git_branch }}"
+    update: yes
+    accept_hostkey: yes
+    force: yes
 
 - name: copy dockerfile to build folder
   tags: deploy
   become: true
   template:
-    src={{ dockerfile }}
-    dest="{{ build_dir }}/{{ name }}"
+    src: "{{ dockerfile }}"
+    dest: "{{ build_dir }}/{{ name }}"
 
 - name: build docker image and tag
   tags: deploy
-  command: sudo docker build {{ build_args | default("") }} --tag="{{ container_image }}:{{ container_tag }}" "{{ build_dir }}/{{ name }}"
+  become: yes
+  command: docker build {{ build_args | default("") }} --tag="{{ container_image }}:{{ container_tag }}" "{{ build_dir }}/{{ name }}"
 
 - name: push docker image
+  become: yes
   when: not do_not_push
-  command: sudo docker push {{ container_image }}:{{ container_tag }}
+  command: docker push {{ container_image }}:{{ container_tag }}
diff --git a/ansible/roles/builder/templates/basic_node/Dockerfile b/ansible/roles/builder/templates/basic_node/Dockerfile
@@ -25,6 +25,12 @@ RUN {{ command }}
 # Add package.json from the current build context (`.` is the repo) second
 ADD ./repo/package.json /{{ name }}/package.json
 
+# Add shrinkwrap from the current build context (`.` is the repo) first
+# If we change a non first-level dependency, this will break cache as expected
+{% if has_shrinkwrap is defined %}
+ADD ./repo/npm-shrinkwrap.json /{{ name }}/npm-shrinkwrap.json
+{% endif %}
+
 # install, should will skip if no package.json change
 WORKDIR /{{ name }}
 RUN npm install --production
diff --git a/ansible/roles/container_kill_start/handlers/main.yml b/ansible/roles/container_kill_start/handlers/main.yml
@@ -2,12 +2,13 @@
 - name: get new container ports
   tags: deploy
   when: hosted_ports is defined
-  shell: sudo docker port {{new_container_id.stdout}} {{ hosted_ports[0] }} | awk --field-separator ':' '{print $2}'
+  become: yes
+  shell: docker port {{new_container_id.stdout}} {{ hosted_ports[0] }} | awk --field-separator ':' '{print $2}'
   register: container_ports
 
 # this assumes only one container is running, ever
 - name: update redis key
   tags: deploy
-  when: container_ports is defined
-  command: sudo docker run --rm redis redis-cli -h {{redis_host_address}} lset {{redis_key}} 1 {{hosted_protocol | default('http') }}://{{ansible_default_ipv4.address}}:{{container_ports.stdout}}
-  when: is_redis_update_required is defined
+  when: container_ports is defined and is_redis_update_required is defined
+  become: yes
+  command: docker run --rm redis redis-cli -h {{redis_host_address}} lset {{redis_key}} 1 {{hosted_protocol | default('http') }}://{{ansible_default_ipv4.address}}:{{container_ports.stdout}}
diff --git a/ansible/roles/container_kill_start/tasks/main.yml b/ansible/roles/container_kill_start/tasks/main.yml
@@ -1,21 +1,23 @@
-- name: "look for stopped {{ container_image }} containers"
+- name: look for stopped {{ container_image }} containers
   tags: deploy
+  become: yes
   shell: >
-    sudo docker ps -a |
+    docker ps -a |
     grep Exited |
     grep {{ container_image }} |
     awk '{print $1}'
   register: stopped_containers
   changed_when: stopped_containers.stdout != ''
 
-- name: "removed stopped {{ container_image }} containers"
+- name: removed stopped {{ container_image }} containers
   tags: deploy
+  become: yes
   when: stopped_containers.changed
-  shell: sudo docker rm {{ item }}
-  with_items: stopped_containers.stdout_lines
+  shell: docker rm {{ item }}
+  with_items: "{{ stopped_containers.stdout_lines }}"
 
 # get current running container with this image
-- name: "look for running containers running {{ container_image }}"
+- name: look for running containers running {{ container_image }}
   tags: deploy
   become: true
   script: findTagRunning.sh {{ container_image }}
@@ -24,16 +26,18 @@
 
 # get latest image
 # tag will default to latest
-- name: "pull down docker image {{ container_image }}:{{ container_tag }}"
+- name: pull down docker image {{ container_image }}:{{ container_tag }}
   tags: deploy
-  command: sudo docker pull {{ container_image }}:{{ container_tag }}
+  become: yes
+  command: docker pull {{ container_image }}:{{ container_tag }}
   when: not do_not_push
 
 # get id of new image
 - name: get id of the docker image
   tags: deploy
+  become: yes
   shell: >
-    sudo docker images
+    docker images
     --no-trunc |
     grep {{ container_image }}.*{{ container_tag }} |
     awk '{print $3}'
@@ -43,32 +47,35 @@
 - name: stop old containers
   tags: deploy
   when: old_containers_id.changed
-  command: sudo docker stop -t {{ stop_time }} {{ item }}
-  with_items: old_containers_id.stdout_lines
+  become: yes
+  command: docker stop -t {{ stop_time }} {{ item }}
+  with_items: "{{ old_containers_id.stdout_lines }}"
 
 # remove the old containers that were running previously
 - name: remove old containers
   tags: deploy
   when: old_containers_id.changed
-  command: sudo docker rm {{ item }}
-  with_items: old_containers_id.stdout_lines
+  become: yes
+  command: docker rm {{ item }}
+  with_items: "{{ old_containers_id.stdout_lines }}"
 
 - name: pause to allow for any odd conditions in system caching tables
   tags: deploy
   when: pause_length_minutes is defined and old_containers_id.changed
-  pause: minutes={{pause_length_minutes}}
+  pause: minutes={{ pause_length_minutes }}
 
 # start our new container with options and args
 - name: start container
   tags: deploy
+  become: yes
   command: >
-    sudo docker run
+    docker run
     --log-driver={{ log_driver }}
     #--log-opt syslog-facility={{ log_facility }}
     #--log-opt tag="{{ log_tag | default ( name ) }}"
     -v {{ app_log_dir }}:{{ app_log_dir }}:rw
     --restart={{ restart_policy | default('always') }}
-    {{ container_run_opts}}
+    {{ container_run_opts }}
     {{ container_image }}:{{ container_tag }}
     {{ container_run_args }}
   register: new_container_id
diff --git a/ansible/roles/container_restart/tasks/main.yml b/ansible/roles/container_restart/tasks/main.yml
@@ -34,4 +34,4 @@
   when: force_stop is defined or (stopped_container.skipped and not running_containers.changed)
   tags: [ deploy ]
   become: yes
-  shell: docker run {{ docker_container_run_opts }} {{ docker_image }}:{{ docker_image_version }}
+  shell: docker run {{ docker_container_run_opts | trim }} {{ docker_image }}:{{ docker_image_version }}
diff --git a/ansible/roles/container_start/handlers/main.yml b/ansible/roles/container_start/handlers/main.yml
diff --git a/ansible/roles/container_start/tasks/main.yml b/ansible/roles/container_start/tasks/main.yml
diff --git a/ansible/roles/eru/tasks/main.yml b/ansible/roles/eru/tasks/main.yml
diff --git a/ansible/roles/eru/templates/11-eru-server.conf b/ansible/roles/eru/templates/11-eru-server.conf
diff --git a/ansible/roles/redis_key/tasks/main.yml b/ansible/roles/redis_key/tasks/main.yml
diff --git a/ansible/roles/ssh-keys/tasks/main.yml b/ansible/roles/ssh-keys/tasks/main.yml
