Merge pull request #477 from CodeNow/navi-lb

Navi proxy + clustering

Author: anandkumarpatel

ansible/group_vars/all.yml

@@ -127,7 +127,6 @@ mongo_hosts: "{% for host in groups['mongodb'] %}{{ hostvars[host]['ansible_defa
 # navi
 navi_host_address: "{{ hostvars[groups['navi'][0]]['ansible_default_ipv4']['address'] }}"
 navi_http_port: 3567
-navi_https_port: 443
 navi_rollbar_token: 719269e87b9b42848472542a8b2059ae
 
 # neo4j

ansible/group_vars/alpha-navi-proxy.yml

@@ -0,0 +1,15 @@
+---
+name: nginx
+
+container_image: "{{ name }}"
+container_tag: "1.10"
+
+container_run_opts: >
+  -d
+  -h {{ name }}
+  -p 0.0.0.0:443:443
+  -p 0.0.0.0:80:80
+  -v /etc/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+  -v /etc/nginx/sites-enabled/:/etc/nginx/sites-enabled/:ro
+  -v /etc/ssl/certs/{{ user_content_domain }}:/etc/ssl/certs/{{ user_content_domain }}:ro
+  -v /var/log/nginx:/var/log/nginx

ansible/group_vars/alpha-navi.yml

@@ -3,7 +3,7 @@ name: navi
 container_image: registry.runnable.com/runnable/{{ name }}
 container_tag: "{{ git_branch }}"
 repo: git@github.com:CodeNow/{{ name }}.git
-hosted_ports: [ "{{ navi_http_port }}", "{{ navi_https_port }}" ]
+hosted_ports: [ "{{ navi_http_port }}" ]
 node_version: "4.2.4"
 npm_version: "2.8.3"
 
@@ -17,11 +17,9 @@ container_envs: >
   -e COOKIE_SECRET={{ navi_cookie_secret }}
   -e DATADOG_HOST={{ datadog_host_address }}
   -e DATADOG_PORT={{ datadog_port }}
-  -e ENABLE_CLUSTERING=1
   -e ENABLE_LRU_CACHE=1
   -e ERROR_URL=http://{{ detention_host_address }}:{{ detention_port }}
   -e HTTP_PORT={{ hosted_ports[0] }}
-  -e HTTPS_PORT={{ hosted_ports[1] }}
   -e LOG_LEVEL_STDOUT=trace
   -e MONGO=mongodb://{{ navi_mongo_host_address }}:{{ navi_mongo_port }}/{{ navi_mongo_database }}
   -e NODE_ENV={{ node_env }}
@@ -32,7 +30,6 @@ container_envs: >
   -e REDIS_CACERT={{ redis_ca_cert_path }}
   -e REDIS_IPADDRESS={{ redis_host_address }}
   -e REDIS_PORT={{ redis_tls_port }}
-  -e USERLAND_IP={{ userland_host_address }}
   {% if navi_intercom_api_key is defined %} -e INTERCOM_API_KEY={{ navi_intercom_api_key }} {% endif %}
   {% if navi_intercom_app_id is defined %} -e INTERCOM_APP_ID={{ navi_intercom_app_id }} {% endif %}
   {% if navi_new_relic_app_name is defined %} -e NEW_RELIC_APP_NAME={{ navi_new_relic_app_name }} {% endif %}
@@ -43,8 +40,6 @@ container_envs: >
 container_run_opts: >
   -h {{ name }}
   -d
-  -p {{ hosted_ports[0] }}:{{ hosted_ports[0] }}
-  -p {{ hosted_ports[1] }}:{{ hosted_ports[1] }}
+  -P
   -v {{ redis_ca_cert_path }}:{{ redis_ca_cert_path }}
-  -v {{ content_domain_certs }}:{{ content_domain_certs }}
   {{ container_envs }}

ansible/group_vars/ec2_sg.yml

@@ -18,7 +18,6 @@ ip_all: "0.0.0.0/0"
 krain_port: 3100
 named_port: 53
 navi_http_port: 3567
-navi_https_port: 443
 neo4j_port: 7473
 neo4j_tls_port: 7474
 redis_port: 6379

ansible/navi-proxy.yml

@@ -0,0 +1,11 @@
+---
+- hosts: userland
+  vars_files:
+    - group_vars/alpha-navi-proxy.yml
+  roles:
+    - role: datadog
+      has_dd_integration: yes
+
+    - role: content-domain-proxy
+
+    - role: container_kill_start

ansible/navi.yml

@@ -4,7 +4,6 @@
 - hosts: mongo-navi
 - hosts: rabbitmq
 - hosts: consul
-- hosts: userland
 
 - hosts: navi
   vars_files:
@@ -13,7 +12,18 @@
   - role: notify
     rollbar_token: "{{ navi_rollbar_token }}"
     tags: [ notify ]
-  - { role: builder, tags: [ build ] }
-  - { role: content-domain-certs }
-  - { role: tls-server-ca, ca_dest: "{{ redis_ca_cert_path }}" }
-  - { role: container_kill_start }
+
+  - role: builder
+    tags: [ build ]
+
+  - role: tls-server-ca
+    ca_dest: "{{ redis_ca_cert_path }}"
+
+  - role: container_start
+    number_of_containers: "{{ ansible_processor_cores }}"
+
+  - role: nginx-proxied-service
+    nginx_host: "{{ groups['userland'][0] }}"
+    target_ip_address: "{{ hostvars[groups['navi'][0]]['ansible_default_ipv4']['address'] }}"
+    templates: [ 69-navi.conf ]
+    nginx_config: proxy

ansible/roles/container_start/tasks/main.yml

@@ -28,7 +28,9 @@
   set_fact:
     number_of_containers: 1
 
-- debug:
+- name: print number of contaienrs
+  tags: deploy
+  debug:
     msg: starting this many containers -- {{ number_of_containers }}
 
 - name: start new container

ansible/roles/content-domain-certs/tasks/main.yml

@@ -0,0 +1,60 @@
+---
+- name: make sure cert directory is in place
+  tags: [ configure_proxy, certs ]
+  become: true
+  file:
+    dest: /etc/ssl/certs/{{ user_content_domain }}
+    state: directory
+
+- name: put certs in place
+  tags: [ configure_proxy, certs ]
+  become: true
+  register: add_certs
+  copy:
+    src: "{{ user_content_domain }}/{{ item }}"
+    dest: /etc/ssl/certs/{{ user_content_domain }}/{{ item }}
+    mode: 0400
+    owner: root
+    group: root
+  with_items:
+  - ca.pem
+  - key.pem
+  - cert.pem
+
+- name: create chained cert
+  tags: [ configure_proxy, certs ]
+  become: true
+  when: add_certs.changed
+  shell: >
+    cat
+    /etc/ssl/certs/{{ user_content_domain }}/cert.pem
+    /etc/ssl/certs/{{ user_content_domain }}/ca.pem
+    >
+    /etc/ssl/certs/{{ user_content_domain }}/chained.pem
+
+- name: create dhparam.pem
+  tags: [ configure_proxy, certs ]
+  become: yes
+  command: openssl dhparam -out /etc/ssl/certs/{{ user_content_domain }}/dhparam.pem 2048
+
+- name: make sure nginx directory is in place
+  tags: [ configure_proxy ]
+  become: true
+  file:
+    dest: /etc/nginx
+    state: directory
+
+
+- name: assert nginx sites-enabled directory
+  tags: [ configure_proxy ]
+  become: yes
+  file:
+    state: directory
+    dest: /etc/nginx/sites-enabled
+
+- name: put nginx configuration in place
+  tags: [ configure_proxy ]
+  become: yes
+  template:
+    src: proxy-nginx.conf
+    dest: /etc/nginx/nginx.conf

ansible/roles/content-domain-proxy/tasks/main.yml

@@ -0,0 +1,29 @@
+user www-data;
+worker_processes 4;
+pid /run/nginx.pid;
+
+events {
+  worker_connections 5000;
+}
+
+http {
+  ##
+  # Basic Settings
+  ##
+  tcp_nodelay on;
+  keepalive_timeout 65;
+  server_tokens off;
+
+  ##
+  # Logging Settings
+  ##
+
+  access_log /var/log/nginx/access.log;
+  error_log /var/log/nginx/error.log;
+
+  ##
+  # Virtual Host Configs
+  ##
+
+  include /etc/nginx/sites-enabled/*;
+}