Merge branch 'master' into SAN-3816-move-logs-or-dont-its-up-to-you

Author: Christopher M. Neill

ansible/delta-hosts/variables

@@ -31,16 +31,16 @@ eru_github_id=46a23f5f99f0aa9460f8
 eru_github_secret=a0336d72e3d540fb9fbbed2c123a81e1cb329dab
 
 [khronos:vars]
+intercom_api_key=14771f14efb617900724a16345e57beb55ba9beb
+intercom_app_id=wqzm3rju
+khronos_canary_github_branches_instance_id=56f05754a443842400b66a53
+khronos_canary_logs_instance_id=56f07f5c1e089a200077f2a3
+khronos_canary_rebuild_instance_id=56df6674058d592500f76c9f
+khronos_canary_rebuild_navi_url=http://canary-build-staging-runnabletest.runnableapp.com:8000/
+khronos_canary_token=24592325a6f881e77a01e3bacf0952c49f4e9f1c
 khronos_mongo_auth=api:72192e5a-a5e1-11e5-add9-0270db32f7ad
 khronos_mongo_database=delta
 khronos_mongo_replset_name=delta
-khronos_canary_token=24592325a6f881e77a01e3bacf0952c49f4e9f1c
-khronos_canary_rebuild_instance_id=56df6674058d592500f76c9f
-khronos_canary_rebuild_navi_url=http://canary-build-staging-runnabletest.runnableapp.com:8000/
-khronos_canary_logs_instance_id=56f07f5c1e089a200077f2a3
-khronos_canary_github_branches_instance_id=56f05754a443842400b66a53
-intercom_app_id=wqzm3rju
-intercom_api_key=14771f14efb617900724a16345e57beb55ba9beb
 
 [metis:vars]
 metis_rollbar_key=fdc8565a7ce64c6d9432c34be425937c

ansible/eru.yml

@@ -1,5 +1,6 @@
 ---
 - hosts: redis
+- hosts: consul
 
 - hosts: eru
   vars_files:

ansible/gamma-hosts/variables

@@ -33,11 +33,6 @@ eru_github_secret=74a23ee56486d57b14f292283cb04625f600917c
 khronos_mongo_auth=api:3f5210b8-8fe3-11e5-8e62-07b6eff19ecb
 khronos_mongo_database=gamma
 khronos_mongo_replset_name=gamma
-khronos_canary_token=d939b612d75244947c9e07b3fec22c9fe95f13ae
-khronos_canary_rebuild_instance_id=56e077e595ba23200046ce7a
-khronos_canary_rebuild_navi_url=http://canary-build-staging-runnabletest.runnable.ninja:8000/
-intercom_app_id=xs5g95pd
-intercom_api_key=037d14f2758eac144de1d2657b2ef9a4f26aef1a
 
 [metis:vars]
 metis_rollbar_key=fdc8565a7ce64c6d9432c34be425937c

ansible/group_vars/all.yml

@@ -33,6 +33,7 @@ loggly_username: ops
 ##
 
 app_log_dir: /var/log
+core_file_dir: /var/log/core
 
 ##
 ## shared application configs
@@ -102,8 +103,6 @@ fluffy_rollbar_token: 6eb108bab1f54687b9b023e5b485967a
 
 # khronos
 rollbar_khronos_token: 5a140828cce14410812d34a3ef936f25
-khronos_canary_logs_instance_id: undefined
-khronos_canary_github_branches_instance_id: undefined
 
 # krain
 krain_port: 3100

ansible/group_vars/alpha-khronos.yml

@@ -25,16 +25,21 @@ cron_container_metrics_command: "/khronos/bin/cli.js --queue khronos:metrics:con
 
 # for container settings
 container_envs: >
-  -e API_URL={{ api_url }}
   -e API_SOCKET_SERVER=https://{{ api_socket_server_hostname }}
-  -e CANARY_API_TOKEN={{ khronos_canary_token }}
-  -e CANARY_REBUILD_INSTANCE_ID={{ khronos_canary_rebuild_instance_id }}
-  -e CANARY_REBUILD_NAVI_URL={{ khronos_canary_rebuild_navi_url }}
+  -e API_URL={{ api_url }}
+  -e CANARY_API_TOKEN={{ khronos_canary_token | default('undefined') }}
+  -e CANARY_GITHUB_BRANCHES_INSTANCE_ID={{ khronos_canary_github_branches_instance_id | default('undefined') }}
+  -e CANARY_LOG_INSTANCE_ID={{ khronos_canary_logs_instance_id | default('undefined') }}
+  -e CANARY_LOG_TERMINAL_SLEEP=10
+  -e CANARY_REBUILD_INSTANCE_ID={{ khronos_canary_rebuild_instance_id | default('undefined') }}
+  -e CANARY_REBUILD_NAVI_URL={{ khronos_canary_rebuild_navi_url | default('undefined') }}
   -e DATADOG_HOST={{ datadog_host_address }}
   -e DATADOG_PORT={{ datadog_port }}
-  -e KHRONOS_PREFETCH=6
+  -e INTERCOM_API_KEY={{ intercom_api_key | default('undefined') }}
+  -e INTERCOM_APP_ID={{ intercom_app_id | default('undefined') }}
   -e KHRONOS_MAVIS=http://{{ mavis_hostname }}
   -e KHRONOS_MONGO=mongodb://{{ khronos_mongo_auth }}@{{ mongo_hosts }}/{{ khronos_mongo_database }}
+  -e KHRONOS_PREFETCH=6
   -e LOG_LEVEL=error
   {% if node_env != "production-epsilon" %}
     -e MONGO_CACERT=/opt/ssl/mongo-client/ca.pem
@@ -46,12 +51,8 @@ container_envs: >
   -e RABBITMQ_PASSWORD={{ rabbit_password }}
   -e RABBITMQ_PORT={{ rabbit_port }}
   -e RABBITMQ_USERNAME={{ rabbit_username }}
+  -e SWARM_HOST=http://{{ swarm_host_address }}:{{ swarm_master_port }}
   -e WORKER_MAX_RETRY_DELAY=3600000
-  -e CANARY_LOG_TERMINAL_SLEEP=10
-  -e CANARY_LOG_INSTANCE_ID={{khronos_canary_logs_instance_id}}
-  -e CANARY_GITHUB_BRANCHES_INSTANCE_ID={{khronos_canary_github_branches_instance_id}}
-  -e INTERCOM_APP_ID={{ intercom_app_id }}
-  -e INTERCOM_API_KEY={{ intercom_api_key }}
 
 container_run_opts: >
   -h {{ name }}

ansible/khronos.yml

@@ -2,6 +2,7 @@
 - hosts: rabbitmq
 - hosts: mongodb
 - hosts: consul
+- hosts: swarm-manager
 
 - hosts: khronos
   vars_files:
@@ -14,4 +15,3 @@
   - { role: builder, tags: [ build ] }
   - { role: khronos, tags: [ deploy ] }
   - { role: container_kill_start }
-

ansible/roles/docker/tasks/main.yml

@@ -16,6 +16,17 @@
     owner=root
     group=root
 
+- name: create core file dir
+  become: true
+  when: docker_config == "runnable" and core_file_dir != "/var/log"
+  tags: coreDump
+  file:
+    path="{{ core_file_dir }}"
+    state=directory
+    user=root
+    group=root
+    mode=0755
+
 - name: change core dump path
   become: true
   when: docker_config == "runnable"
@@ -24,7 +35,7 @@
     name=kernel.core_pattern
     reload=true
     state=present
-    value="/var/log/core.%h.%e.%t"
+    value="{{ core_file_dir }}/core.%h.%e.%t"
 
 - name: install aufs with linux-image-extra-{{ ansible_kernel }}
   become: true

ansible/roles/khronos/tasks/main.yml

@@ -33,6 +33,7 @@
   - name: Khronos CLI - Github Branch Canary
     minute: "*/5"
     script: github-branch-canary.sh
+    state: "{% if node_env == 'production-delta' %}present{% else %}absent{% endif %}"
   - name: Khronos CLI - Log Canary
     minute: "*/5"
     script: log-canary.sh