Merge branch 'master' into SAN-3546-loggly-tags

Christopher M. Neill · Christopher M. Neill · commit 6bd403fb80d5 · 2016-04-13T13:50:25.000-07:00
diff --git a/ansible/delta-hosts/variables b/ansible/delta-hosts/variables
@@ -31,16 +31,16 @@ eru_github_id=46a23f5f99f0aa9460f8
 eru_github_secret=a0336d72e3d540fb9fbbed2c123a81e1cb329dab
 
 [khronos:vars]
+intercom_api_key=14771f14efb617900724a16345e57beb55ba9beb
+intercom_app_id=wqzm3rju
+khronos_canary_github_branches_instance_id=56f05754a443842400b66a53
+khronos_canary_logs_instance_id=56f07f5c1e089a200077f2a3
+khronos_canary_rebuild_instance_id=56df6674058d592500f76c9f
+khronos_canary_rebuild_navi_url=http://canary-build-staging-runnabletest.runnableapp.com:8000/
+khronos_canary_token=24592325a6f881e77a01e3bacf0952c49f4e9f1c
 khronos_mongo_auth=api:72192e5a-a5e1-11e5-add9-0270db32f7ad
 khronos_mongo_database=delta
 khronos_mongo_replset_name=delta
-khronos_canary_token=24592325a6f881e77a01e3bacf0952c49f4e9f1c
-khronos_canary_rebuild_instance_id=56df6674058d592500f76c9f
-khronos_canary_rebuild_navi_url=http://canary-build-staging-runnabletest.runnableapp.com:8000/
-khronos_canary_logs_instance_id=56f07f5c1e089a200077f2a3
-khronos_canary_github_branches_instance_id=56f05754a443842400b66a53
-intercom_app_id=wqzm3rju
-intercom_api_key=14771f14efb617900724a16345e57beb55ba9beb
 
 [metis:vars]
 metis_rollbar_key=fdc8565a7ce64c6d9432c34be425937c
diff --git a/ansible/eru.yml b/ansible/eru.yml
@@ -1,5 +1,6 @@
 ---
 - hosts: redis
+- hosts: consul
 
 - hosts: eru
   vars_files:
diff --git a/ansible/gamma-hosts/variables b/ansible/gamma-hosts/variables
@@ -33,11 +33,6 @@ eru_github_secret=74a23ee56486d57b14f292283cb04625f600917c
 khronos_mongo_auth=api:3f5210b8-8fe3-11e5-8e62-07b6eff19ecb
 khronos_mongo_database=gamma
 khronos_mongo_replset_name=gamma
-khronos_canary_token=d939b612d75244947c9e07b3fec22c9fe95f13ae
-khronos_canary_rebuild_instance_id=56e077e595ba23200046ce7a
-khronos_canary_rebuild_navi_url=http://canary-build-staging-runnabletest.runnable.ninja:8000/
-intercom_app_id=xs5g95pd
-intercom_api_key=037d14f2758eac144de1d2657b2ef9a4f26aef1a
 
 [metis:vars]
 metis_rollbar_key=fdc8565a7ce64c6d9432c34be425937c
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
@@ -28,6 +28,13 @@ loggly_token: f673760d-e0b3-4a93-a15e-2862ea074f91
 loggly_password: TebejAcutHeH_Ch0tR9ru4anaT6CRu*3
 loggly_username: ops
 
+##
+## cores and logs
+##
+
+app_log_dir: /var/log
+core_file_dir: /var/log/core
+
 ##
 ## shared application configs
 ##
@@ -96,8 +103,6 @@ fluffy_rollbar_token: 6eb108bab1f54687b9b023e5b485967a
 
 # khronos
 rollbar_khronos_token: 5a140828cce14410812d34a3ef936f25
-khronos_canary_logs_instance_id: undefined
-khronos_canary_github_branches_instance_id: undefined
 
 # krain
 krain_port: 3100
diff --git a/ansible/group_vars/alpha-khronos.yml b/ansible/group_vars/alpha-khronos.yml
@@ -8,33 +8,41 @@ npm_version: 2
 
 # for cron job
 # this is the list of queues we want to enqueue a job into
-cron_queues: khronos:containers:orphan:prune khronos:context-versions:prune-expired khronos:images:prune khronos:weave:prune
+main_cron_queues: "\
+  khronos:containers:orphan:prune \
+  khronos:context-versions:prune-expired \
+  khronos:images:prune \
+  khronos:weave:prune"
+
+canary_cron_queues: "\
+  khronos:canary:build \
+  khronos:canary:github-branch \
+  khronos:canary:log \
+  khronos:canary:network \
+  khronos:metrics:container-status"
+
 # a nice version of the rabbitmq host
 cron_rabbit_host_address: "{{ rabbit_host_address }}:{{ rabbit_port }}"
 # a quick version of authentication for rabbit for cron
 cron_rabbit_auth: --username {{ rabbit_username }} --password {{ rabbit_password }}
-# this is the command we want to run in the container.
-cron_command: "bash -c \"for QUEUE in {{ cron_queues }}; do /khronos/bin/cli.js --queue \\$QUEUE --job '{}' --host {{ cron_rabbit_host_address }} {{ cron_rabbit_auth }}; done\""
-# This is the build canary cron command
-cron_build_canary_command: "/khronos/bin/cli.js --queue khronos:canary:build --job '{}' --host {{ cron_rabbit_host_address }} {{ cron_rabbit_auth }}"
-# This is the log canary cron command
-cron_log_canary_command: "/khronos/bin/cli.js --queue khronos:canary:log --job '{}' --host {{ cron_rabbit_host_address }} {{ cron_rabbit_auth }}"
-# This is the github-branch canary cron command
-cron_github_branch_canary_command: "/khronos/bin/cli.js --queue khronos:canary:github-branch --job '{}' --host {{ cron_rabbit_host_address }} {{ cron_rabbit_auth }}"
-cron_container_metrics_command: "/khronos/bin/cli.js --queue khronos:metrics:container-status --job '{}' --host {{ cron_rabbit_host_address }} {{ cron_rabbit_auth }}"
 
 # for container settings
 container_envs: >
-  -e API_URL={{ api_url }}
   -e API_SOCKET_SERVER=https://{{ api_socket_server_hostname }}
-  -e CANARY_API_TOKEN={{ khronos_canary_token }}
-  -e CANARY_REBUILD_INSTANCE_ID={{ khronos_canary_rebuild_instance_id }}
-  -e CANARY_REBUILD_NAVI_URL={{ khronos_canary_rebuild_navi_url }}
+  -e API_URL={{ api_url }}
+  -e CANARY_API_TOKEN={{ khronos_canary_token | default('undefined') }}
+  -e CANARY_GITHUB_BRANCHES_INSTANCE_ID={{ khronos_canary_github_branches_instance_id | default('undefined') }}
+  -e CANARY_LOG_INSTANCE_ID={{ khronos_canary_logs_instance_id | default('undefined') }}
+  -e CANARY_LOG_TERMINAL_SLEEP=10
+  -e CANARY_REBUILD_INSTANCE_ID={{ khronos_canary_rebuild_instance_id | default('undefined') }}
+  -e CANARY_REBUILD_NAVI_URL={{ khronos_canary_rebuild_navi_url | default('undefined') }}
   -e DATADOG_HOST={{ datadog_host_address }}
   -e DATADOG_PORT={{ datadog_port }}
-  -e KHRONOS_PREFETCH=6
+  -e INTERCOM_API_KEY={{ intercom_api_key | default('undefined') }}
+  -e INTERCOM_APP_ID={{ intercom_app_id | default('undefined') }}
   -e KHRONOS_MAVIS=http://{{ mavis_hostname }}
   -e KHRONOS_MONGO=mongodb://{{ khronos_mongo_auth }}@{{ mongo_hosts }}/{{ khronos_mongo_database }}
+  -e KHRONOS_PREFETCH=6
   -e LOG_LEVEL=error
   {% if node_env != "production-epsilon" %}
     -e MONGO_CACERT=/opt/ssl/mongo-client/ca.pem
@@ -46,12 +54,8 @@ container_envs: >
   -e RABBITMQ_PASSWORD={{ rabbit_password }}
   -e RABBITMQ_PORT={{ rabbit_port }}
   -e RABBITMQ_USERNAME={{ rabbit_username }}
+  -e SWARM_HOST=http://{{ swarm_host_address }}:{{ swarm_master_port }}
   -e WORKER_MAX_RETRY_DELAY=3600000
-  -e CANARY_LOG_TERMINAL_SLEEP=10
-  -e CANARY_LOG_INSTANCE_ID={{khronos_canary_logs_instance_id}}
-  -e CANARY_GITHUB_BRANCHES_INSTANCE_ID={{khronos_canary_github_branches_instance_id}}
-  -e INTERCOM_APP_ID={{ intercom_app_id }}
-  -e INTERCOM_API_KEY={{ intercom_api_key }}
 
 container_run_opts: >
   -h {{ name }}
@@ -60,5 +64,3 @@ container_run_opts: >
   -v /opt/ssl/mongo-client:/opt/ssl/mongo-client:ro
   --memory=1500m
   {{container_envs}}
-
-npm_start_command: run start-json-output
diff --git a/ansible/group_vars/alpha-vault.yml b/ansible/group_vars/alpha-vault.yml
@@ -14,7 +14,7 @@ container_run_opts: >
   -v /opt/runnable/vault/vault.hcl:/vault.hcl:ro
   -v /opt/vault/client-consul:/opt/vault/client-consul:ro
   -v /opt/vault/server:/opt/vault/server:ro
-  -v /var/log:/var/log:rw
+  -v {{ app_log_dir }}:{{ app_log_dir }}:rw
   -p {{ ansible_default_ipv4.address }}:8200:8200
   -p {{ ansible_default_ipv4.address }}:8201:8201
   --cap-add IPC_LOCK
diff --git a/ansible/khronos.yml b/ansible/khronos.yml
@@ -2,6 +2,7 @@
 - hosts: rabbitmq
 - hosts: mongodb
 - hosts: consul
+- hosts: swarm-manager
 
 - hosts: khronos
   vars_files:
@@ -14,4 +15,3 @@
   - { role: builder, tags: [ build ] }
   - { role: khronos, tags: [ deploy ] }
   - { role: container_kill_start }
-
diff --git a/ansible/link.yml b/ansible/link.yml
@@ -1,6 +1,7 @@
 ---
 - hosts: mongo-navi
 - hosts: rabbitmq
+- hosts: consul
 
 - hosts: link
   vars_files:
diff --git a/ansible/roles/builder/templates/basic_node/Dockerfile b/ansible/roles/builder/templates/basic_node/Dockerfile
@@ -22,12 +22,6 @@ RUN {{ command }}
 {% endfor %}
 {% endif %}
 
-# Add shrinkwrap from the current build context (`.` is the repo) first
-# If we change a non first-level dependency, this will break cache as expected
-{% if name in ['api', 'api-worker'] %}
-ADD ./repo/npm-shrinkwrap.json /{{ name }}/npm-shrinkwrap.json
-{% endif %}
-
 # Add package.json from the current build context (`.` is the repo) second
 ADD ./repo/package.json /{{ name }}/package.json
 
@@ -45,4 +39,4 @@ RUN {{ command }}
 {% endif %}
 
 # Define default command.
-CMD ulimit -c unlimited && /usr/local/bin/npm {{ npm_start_command | default('start') }} > /var/log/{{ name }}.log 2>&1
+CMD ulimit -c unlimited && /usr/local/bin/npm {{ npm_start_command | default('start') }} > {{ app_log_dir }}/{{ name }}.log 2>&1
diff --git a/ansible/roles/container_kill_start/tasks/main.yml b/ansible/roles/container_kill_start/tasks/main.yml
@@ -64,7 +64,8 @@
   command: >
     sudo docker run
     --log-driver={{ log_driver | default("none")}}
-    -v /var/log:/var/log:rw
+    -v {{ app_log_dir }}:{{ app_log_dir }}:rw
+    -v /dev/log:/dev/log:rw
     --restart={{ restart_policy | default('always') }}
     {{ container_run_opts}}
     {{ container_image }}:{{ container_tag }}
diff --git a/ansible/roles/container_start/tasks/main.yml b/ansible/roles/container_start/tasks/main.yml
@@ -28,7 +28,7 @@
 
 - name: start new container
   tags: deploy
-  shell: for i in {{ '{' }}1..{{ number_of_containers }}{{ '}' }}; do sudo docker run --log-driver=none -v /var/log:/var/log:rw --restart={{ restart_policy | default('no') }} {{ container_run_opts }} {{ container_image }}:{{ container_tag }} {{ container_run_args }}; done
+  shell: for i in {{ '{' }}1..{{ number_of_containers }}{{ '}' }}; do sudo docker run --log-driver=none -v {{ app_log_dir }}:{{ app_log_dir }}:rw -v /dev/log://dev/log:rw --restart={{ restart_policy | default('no') }} {{ container_run_opts }} {{ container_image }}:{{ container_tag }} {{ container_run_args }}; done
   args:
     executable: /bin/bash
   register: new_container_ids
diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml
@@ -16,6 +16,17 @@
     owner=root
     group=root
 
+- name: create core file dir
+  become: true
+  when: docker_config == "runnable" and core_file_dir != "/var/log"
+  tags: coreDump
+  file:
+    path="{{ core_file_dir }}"
+    state=directory
+    owner=root
+    group=root
+    mode=0755
+
 - name: change core dump path
   become: true
   when: docker_config == "runnable"
@@ -24,7 +35,7 @@
     name=kernel.core_pattern
     reload=true
     state=present
-    value="/var/log/core.%h.%e.%t"
+    value="{{ core_file_dir }}/core.%h.%e.%t"
 
 - name: install aufs with linux-image-extra-{{ ansible_kernel }}
   become: true
@@ -80,6 +91,15 @@
     update_cache=yes
     cache_valid_time=604800
 
+- name: enforce modes on /docker
+  become: true
+  file:
+    path=/docker
+    state=directory
+    owner=root
+    group=root
+    mode=0755
+
 - name: restart docker
   when: (copied_config.changed and restart is defined) or dock is defined
   command: sudo service docker restart
diff --git a/ansible/roles/khronos/README.md b/ansible/roles/khronos/README.md
@@ -1,3 +1,3 @@
 Run Khronos CLI tool out of cron once daily.
 
-`tasks/main.yml` - install cron entry to run `/khronos/bin/cli.js` and output to `/var/log/khonos_cron.log`
+`tasks/main.yml` - install cron entry to run `/khronos/bin/cli.js` and output to `{{ app_log_dir }}/khonos_cron.log`
diff --git a/ansible/roles/khronos/tasks/main.yml b/ansible/roles/khronos/tasks/main.yml
@@ -2,44 +2,30 @@
   become: yes
   tags: cron
   template:
-    src: "{{ item }}"
+    src: "khronos-cron.sh"
     mode: 0744
-    dest: /opt/runnable/{{ item }}
+    dest: /opt/runnable/{{ item.script }}
   with_items:
-  - khronos-cron.sh
-  - build-canary.sh
-  - log-canary.sh
-  - github-branch-canary.sh
-  - container-metrics.sh
+  - { cron_queues: "{{ main_cron_queues }}", script: "main-cron.sh"}
+  - { cron_queues: "{{ canary_cron_queues }}", script: "canary-cron.sh"}
 
 - name: Put Khronos script into crontab
   become: yes
   tags: cron
   cron:
     name: "{{ item.name }}"
-    job: /opt/runnable/{{ item.script }} >> /var/log/{{ item.script }}.log 2>&1
+    job: /opt/runnable/{{ item.script }} >> {{ app_log_dir }}/{{ item.script }}.log 2>&1
     minute: "{{ item.minute | default('*') }}"
     hour: "{{ item.hour | default('*') }}"
     state: "{{ item.state | default('present') }}"
   with_items:
   - name: Khronos CLI - Daily Cleanup
     minute: 13
     hour: 1,7,13,19
-    script: khronos-cron.sh
+    script: main-cron.sh
   - name: Khronos CLI - Canary
     minute: "*/5"
-    script: build-canary.sh
-    state: "{% if node_env == 'production-delta' %}present{% else %}absent{% endif %}"
-  - name: Khronos CLI - Github Branch Canary
-    minute: "*/5"
-    script: github-branch-canary.sh
-  - name: Khronos CLI - Log Canary
-    minute: "*/5"
-    script: log-canary.sh
-    state: "{% if node_env == 'production-delta' %}present{% else %}absent{% endif %}"
-  - name: Container Metrics
-    minute: "*/5"
-    script: container-metrics.sh
+    script: canary-cron.sh
     state: "{% if node_env == 'production-delta' %}present{% else %}absent{% endif %}"
 
 - name: make directory for mongo certificates
diff --git a/ansible/roles/khronos/templates/build-canary.sh b/ansible/roles/khronos/templates/build-canary.sh
diff --git a/ansible/roles/khronos/templates/container-metrics.sh b/ansible/roles/khronos/templates/container-metrics.sh
diff --git a/ansible/roles/khronos/templates/github-branch-canary.sh b/ansible/roles/khronos/templates/github-branch-canary.sh
diff --git a/ansible/roles/khronos/templates/khronos-cron.sh b/ansible/roles/khronos/templates/khronos-cron.sh
@@ -2,4 +2,7 @@
 # THIS FILE IS MANAGED BY ANSIBLE. PLEASE DO NOT MODIFY. MODIFICATIONS WILL NOT BE SAVED.
 # AUTHOR: BRYAN KENDALL
 
-docker run --rm {{ container_image }}:{{ container_tag }} {{ cron_command }}
+docker run --rm {{ container_image }}:{{ container_tag }} bash -c " \
+  for QUEUE in {{ item.cron_queues }}; do \
+    /khronos/bin/cli.js --queue \$QUEUE --job '{}' --host {{ cron_rabbit_host_address }} {{ cron_rabbit_auth }}; \
+  done"
diff --git a/ansible/roles/khronos/templates/log-canary.sh b/ansible/roles/khronos/templates/log-canary.sh
diff --git a/ansible/roles/log-mkdir/tasks/main.yml b/ansible/roles/log-mkdir/tasks/main.yml
@@ -0,0 +1,12 @@
+---
+- name: create application logs directory
+  tags: rsyslog
+  when: app_log_dir != "/var/log"
+  become: true
+  file:
+    path="{{ app_log_dir }}"
+    state=directory
+    owner=root
+    group=syslog
+    mode=0775
+
diff --git a/ansible/roles/loggly-rotate/tasks/main.yml b/ansible/roles/loggly-rotate/tasks/main.yml
@@ -98,7 +98,7 @@
   tags: [loggly, deploy]
   become: true
   file:
-    path=/var/log/{{ name }}-daemon.log
+    path={{ app_log_dir }}/{{ name }}-daemon.log
     state=touch
     owner=syslog
     group=adm
diff --git a/ansible/roles/loggly-rotate/templates/21-output-syslog.conf.j2 b/ansible/roles/loggly-rotate/templates/21-output-syslog.conf.j2
@@ -2,7 +2,7 @@ $WorkDirectory /var/spool/rsyslog
 
 # start log rotation via outchannel
 # outchannel definition
-$outchannel log_rotation_{{ name }},/var/log/{{ name }}-daemon.log,524288000,/opt/runnable/bin/rotate-{{ name }}-logs.sh
+$outchannel log_rotation_{{ name }},{{ app_log_dir }}/{{ name }}-daemon.log,524288000,/opt/runnable/bin/rotate-{{ name }}-logs.sh
 #  activate the channel and log everything to it
 if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then :omfile:$log_rotation_{{ name }}
 # end log rotation via outchannel
diff --git a/ansible/roles/loggly-rotate/templates/rotate-logs.sh.j2 b/ansible/roles/loggly-rotate/templates/rotate-logs.sh.j2
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # literally truncate the file
-cat /dev/null > /var/log/{{ name }}-daemon.log 2>&1
+cat /dev/null > {{ app_log_dir }}/{{ name }}-daemon.log 2>&1
 # exit code? don't care
 
 exit 0
diff --git a/ansible/roles/loggly/tasks/main.yml b/ansible/roles/loggly/tasks/main.yml
diff --git a/ansible/roles/loggly/templates/21-filemonitoring-app.conf.j2 b/ansible/roles/loggly/templates/21-filemonitoring-app.conf.j2
diff --git a/ansible/roles/node_service/templates/upstart.conf b/ansible/roles/node_service/templates/upstart.conf
diff --git a/ansible/roles/rabbitmq/templates/rabbitmq.config b/ansible/roles/rabbitmq/templates/rabbitmq.config
