Merge pull request #487 from CodeNow/SAN-3868-Failover-Canary

Create hourly canary script for Failover

Author: anandkumarpatel

ansible/delta-hosts/variables

@@ -50,6 +50,7 @@ khronos_canary_logs_instance_id=56f07f5c1e089a200077f2a3
 khronos_canary_rebuild_instance_id=571b39b9d35173300021667d
 khronos_canary_rebuild_navi_url=http://canary-build-staging-runnabletest.runnableapp.com:8000/
 khronos_canary_token=24592325a6f881e77a01e3bacf0952c49f4e9f1c
+khronos_canary_failover_token=84549e76545306de61d47f23b1d1831e1c95a400
 khronos_mongo_auth=api:72192e5a-a5e1-11e5-add9-0270db32f7ad
 khronos_mongo_database=delta
 khronos_mongo_replset_name=delta

ansible/group_vars/alpha-khronos.yml

@@ -22,6 +22,9 @@ canary_cron_queues: "\
   khronos:canary:network \
   khronos:metrics:container-status"
 
+hourly_canary_cron_queues: "\
+  khronos:canary:failover \"
+
 # a nice version of the rabbitmq host
 cron_rabbit_host_address: "{{ rabbit_host_address }}:{{ rabbit_port }}"
 # a quick version of authentication for rabbit for cron
@@ -33,6 +36,9 @@ container_envs: >
   -e API_URL={{ api_url }}
   -e CONSUL_HOST={{ consul_host_address }}:{{ consul_api_port }}
   -e CANARY_API_TOKEN={{ khronos_canary_token | default('undefined') }}
+  {% if khronos_canary_failover_token is defined %}
+    -e CANARY_API_FAILOVER_TOKEN={{ khronos_canary_failover_token }}
+  {% endif %}
   -e CANARY_GITHUB_BRANCHES_INSTANCE_ID={{ khronos_canary_github_branches_instance_id | default('undefined') }}
   -e CANARY_LOG_INSTANCE_ID={{ khronos_canary_logs_instance_id | default('undefined') }}
   -e CANARY_LOG_TERMINAL_SLEEP=10
@@ -57,6 +63,7 @@ container_envs: >
   -e RABBITMQ_PORT={{ rabbit_port }}
   -e RABBITMQ_USERNAME={{ rabbit_username }}
   -e SWARM_HOST=http://{{ swarm_host_address }}:{{ swarm_master_port }}
+  -e USER_CONTENT_DOMAIN={{ user_content_domain }}
   -e WORKER_MAX_RETRY_DELAY=3600000
 
 container_run_opts: >

ansible/roles/khronos/tasks/main.yml

@@ -8,6 +8,7 @@
   with_items:
   - { cron_queues: "{{ main_cron_queues }}", script: "main-cron.sh"}
   - { cron_queues: "{{ canary_cron_queues }}", script: "canary-cron.sh"}
+  - { cron_queues: "{{ hourly_canary_cron_queues }}", script: "hourly-canary-cron.sh"}
 
 - name: Put Khronos script into crontab
   become: yes
@@ -27,6 +28,10 @@
     minute: "*/5"
     script: canary-cron.sh
     state: "{% if node_env == 'production-delta' %}present{% else %}absent{% endif %}"
+  - name: Khronos CLI - Hourly Canary
+    minute: "*/50"
+    script: hourly-canary-cron.sh
+    state: "{% if node_env == 'production-delta' %}present{% else %}absent{% endif %}"
 
 - name: make directory for mongo certificates
   become: yes