merge with master

Anandkumar Patel · Anandkumar Patel · commit cb3540e5a8e8 · 2016-01-12T11:26:38.000-08:00
diff --git a/README.md b/README.md
@@ -105,4 +105,8 @@ It is the custom at Runnable to play a song to the entire team when deploying. F
 
 [Sauron: Sauron theme song from LOTR](https://www.youtube.com/watch?v=V_rk9VBrXMY)
 
+[Detention: Unbreakable Kimmy Schmidt](https://youtu.be/CV9xF8CjhJk?t=21s)
+
+[Link: Zelda Main Theme Song](https://www.youtube.com/watch?v=cGufy1PAeTU)
+
 **IMPORTANT:** Make sure the play the song loud and proud when deploying!
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
@@ -0,0 +1,3 @@
+[defaults]
+# Required so `sudo: yes` does not lose the environment variables, which hold the ssh-agent socket
+sudo_flags=-HE
diff --git a/ansible/delta-hosts/hosts b/ansible/delta-hosts/hosts
@@ -29,7 +29,7 @@ delta-web
 delta-services
 
 [worker]
-delta-api
+delta-api-worker
 
 [docks]
 
@@ -97,6 +97,7 @@ bastion
 hipache
 mongodb
 api
+worker
 web
 redis
 redis-slave
diff --git a/ansible/delta-hosts/variables b/ansible/delta-hosts/variables
@@ -21,7 +21,7 @@ eru_github_id=46a23f5f99f0aa9460f8
 eru_github_secret=a0336d72e3d540fb9fbbed2c123a81e1cb329dab
 
 [khronos:vars]
-khronos_mongo_auth=api:oW4c7x9Wiv28oiNBy2Bc
+khronos_mongo_auth=api:72192e5a-a5e1-11e5-add9-0270db32f7ad
 khronos_mongo_database=delta
 khronos_mongo_replset_name=delta
 
@@ -59,8 +59,9 @@ domain=runnable.io
 mongo_port=27017
 new_relic_license_key=338516e0826451c297d44dc60aeaf0a0ca4bfead
 node_env=production-delta
-pg_host=delta-infrastructure-db.cnksgdqarobf.us-west-2.rds.amazonaws.com
+pg_host=delta-infradb.cnksgdqarobf.us-west-2.rds.amazonaws.com
 pg_pass=59a5524e-a772-11e5-bedc-1bdc0db458b3
+pg_user=astral
 rabbit_password=wKK7g7NWKpQXEeSzyWB7mIpxZIL8H2mDSf3Q6czR3Vk
 rabbit_username=o2mdLh9N9Ke2GzhoK8xsruYPhIQFN7iEL44dQJoq7OM
 registry_host=10.8.4.126
diff --git a/ansible/gamma-hosts/hosts b/ansible/gamma-hosts/hosts
@@ -6,9 +6,9 @@ gamma-hipache httpsCheckForBackend80=false prependIncomingPort=true subDomainDep
 gamma-userland-hipache domain=runnable.ninja httpsCheckForBackend80=true prependIncomingPort=true subDomainDepth=3
 
 [mongodb]
-gamma-mongo-us-west-2a
-gamma-mongo-us-west-2b
-gamma-mongo-us-west-2c
+gamma-mongo-a
+gamma-mongo-b
+gamma-mongo-c
 
 [neo4j]
 gamma-neo4j
@@ -29,7 +29,7 @@ gamma-web
 gamma-services
 
 [worker]
-gamma-api
+gamma-api-worker
 
 [eru]
 gamma-services
@@ -85,6 +85,9 @@ gamma-registry
 [swarm-manager]
 gamma-services
 
+[metis]
+gamma-services
+
 [docks]
 
 [dock]
@@ -104,13 +107,17 @@ navi
 neo4j
 optimus
 rabbitmq
+api
+worker
+web
 redis
 redis-slave
 registry
 sauron
 shiva
 swarm-manager
 web
+metis
 
 [ec2]
 127.0.0.1
diff --git a/ansible/gamma-hosts/variables b/ansible/gamma-hosts/variables
@@ -21,7 +21,7 @@ eru_github_id=8abb08f83f6d1c52bd1a
 eru_github_secret=74a23ee56486d57b14f292283cb04625f600917c
 
 [khronos:vars]
-khronos_mongo_auth=api:oW4c7x9Wiv28oiNBy2Bc
+khronos_mongo_auth=api:3f5210b8-8fe3-11e5-8e62-07b6eff19ecb
 khronos_mongo_database=gamma
 khronos_mongo_replset_name=gamma
 
@@ -54,6 +54,8 @@ vault_aws_region=us-east-1
 ansible_ssh_private_key_file=~/.ssh/gamma.pem
 datadog_host_address=10.4.6.251
 datadog_tags=env:gamma
+datadog_mongodb_user=datadog
+datadog_mongodb_pwd=sqa3WBgkCgZsFZuex0kBNahZ
 domain=runnable-gamma.com
 mongo_port=27017
 new_relic_license_key=338516e0826451c297d44dc60aeaf0a0ca4bfead
@@ -66,6 +68,7 @@ registry_host=10.4.4.82
 swarm_token=d363b783f03a845a2c82b081bfe8443e
 user_content_domain=runnable.ninja
 api_hello_runnable_github_token=88ddc423c2312d02a8bbcaad76dd4c374a30e4af
+no_dd_monitoring=false
 vault_auth_token=e22c3ebc-11cf-653b-7df0-79d78a499458
 vault_token_01=71d7b4754686013c8b9cfb22bafae79c661849dcd67c483c89efba12c0466aa201
 vault_token_02=794d6f7a3459c332a1fd2bbcc9230a7f84f1639806039ee8be547828cd7ab03a02
diff --git a/ansible/group_vars/alpha-mongo.yml b/ansible/group_vars/alpha-mongo.yml
@@ -7,3 +7,6 @@ db_path: "/opt/mongodb/db"
 container_image: mongo
 container_tag: "latest"
 container_run_opts: "-d --name mongodb -p 27017:27017 -v {{ db_path }}:/data/db:rw"
+
+# do not monitor docker-daemon for mongo, as there is none
+no_dd_monitoring: true
diff --git a/ansible/group_vars/alpha-navi.yml b/ansible/group_vars/alpha-navi.yml
@@ -8,14 +8,19 @@ node_version: "4.1.0"
 npm_version: "2.8.3"
 
 container_envs: >
-  -e CLUSTER_WORKERS=20
   -e DATADOG_HOST={{ datadog_host_address }}
   -e DATADOG_PORT={{ datadog_port }}
+  -e ENABLE_CLUSTERING=1
+  -e ENABLE_LRU_CACHE=1
   -e ERROR_URL=http://{{ detention_host_address }}:{{ detention_port }}
   -e HTTP_PORT={{ hosted_ports[0] }}
   -e LOG_LEVEL_STDOUT=trace
   -e MONGO=mongodb://{{ navi_mongo_host_address }}:{{ navi_mongo_port }}/{{ navi_mongo_database }}
   -e NODE_ENV={{ node_env }}
+  -e RABBITMQ_HOSTNAME={{ rabbit_host_address }}
+  -e RABBITMQ_PASSWORD={{ rabbit_password }}
+  -e RABBITMQ_PORT={{ rabbit_port }}
+  -e RABBITMQ_USERNAME={{ rabbit_username }}
   -e REDIS_IPADDRESS={{ redis_host_address }}
 
 container_run_opts: >
diff --git a/ansible/group_vars/alpha-rabbitmq.yml b/ansible/group_vars/alpha-rabbitmq.yml
@@ -10,3 +10,5 @@ container_tag: "3.5.3-management"
 container_run_opts: "-d --name prod-rabbit -v {{ db_path }}:/var/lib/rabbitmq:rw -v {{ config_path }}/rabbitmq.config:/etc/rabbitmq/rabbitmq.config:ro -p 54320:15672 -p 54321:5672"
 
 hosted_ports: [ 5672, 15672 ]
+# monitor the docker-daemon
+no_dd_monitoring: false
diff --git a/ansible/group_vars/alpha-sauron.yml b/ansible/group_vars/alpha-sauron.yml
@@ -13,7 +13,7 @@ docker_path: "/bin/docker"
 dockerfile_pre_install_commands: [
   "curl -L https://get.docker.com/builds/Linux/x86_64/docker-1.6.2 -o {{ docker_path }}",
   "chmod a+x {{ docker_path }}",
-  "curl -L https://github.com/weaveworks/weave/releases/download/v1.3.1/weave -o {{ weave_path }}",
+  "curl -L https://github.com/weaveworks/weave/releases/download/v1.4.1/weave -o {{ weave_path }}",
   "chmod a+x {{ weave_path }}"
 ]
 
diff --git a/ansible/mongo.yml b/ansible/mongo.yml
@@ -6,3 +6,4 @@
   - { role: notify, tags: "notify" }
   - { role: database }
   - { role: container_kill_start, tags: "deploy" }
+  - { role: datadog, tags: "datadog" }
diff --git a/ansible/navi.yml b/ansible/navi.yml
@@ -2,6 +2,7 @@
 - hosts: redis
 - hosts: detention
 - hosts: mongo-navi
+- hosts: rabbitmq
 
 - hosts: navi
   vars_files:
diff --git a/ansible/rabbitmq.yml b/ansible/rabbitmq.yml
@@ -7,3 +7,4 @@
   - { role: database }
   - { role: rabbitmq }
   - { role: container_kill_start, tags: "deploy" }
+  - { role: datadog, tags: "datadog" }
diff --git a/ansible/roles/builder/tasks/main.yml b/ansible/roles/builder/tasks/main.yml
@@ -6,6 +6,16 @@
     path="{{ build_dir }}/{{ name }}"
     state=directory
 
+- name: pull the git repository
+  sudo: yes
+  git:
+    repo="{{ repo }}"
+    dest="{{ build_dir }}/{{ name }}/repo"
+    version="{{ git_branch }}"
+    update=yes
+    accept_hostkey=True
+    force=yes
+
 - name: copy dockerfile to build folder
   sudo: yes
   template:
diff --git a/ansible/roles/builder/templates/basic_node/Dockerfile b/ansible/roles/builder/templates/basic_node/Dockerfile
@@ -15,18 +15,22 @@ ENV {{ env }}
 # setup node and npm versions
 RUN n {{ node_version }} && npm install -g npm@{{ npm_version }}
 
-# Download Repo
-RUN git clone -b {{ git_branch }} --single-branch {{ repo }} /{{ name }}
-
-WORKDIR /{{ name }}
 {% if dockerfile_pre_install_commands is defined %}
 {% for command in dockerfile_pre_install_commands %}
 RUN {{ command }}
 {% endfor %}
 {% endif %}
 
+# Add package.json from the current build context (`.` is the repo) first
+ADD ./repo/package.json /{{ name }}/package.json
+
+# install, should will skip if no package.json change
+WORKDIR /{{ name }}
 RUN npm install --production
 
+# move the current build context (`.` is the repo) to /{{ name }}
+ADD ./repo /{{ name }}
+
 {% if dockerfile_post_install_commands is defined %}
 {% for command in dockerfile_post_install_commands %}
 RUN {{ command }}
@@ -35,4 +39,3 @@ RUN {{ command }}
 
 # Define default command.
 CMD ulimit -c unlimited && /usr/local/bin/npm {{ npm_start_command | default('start') }} > /var/log/{{ name }}.log 2>&1
-
diff --git a/ansible/roles/datadog/tasks/main.yml b/ansible/roles/datadog/tasks/main.yml
@@ -1,46 +1,78 @@
 ---
 - name: add https transport
-  tags: ['datadog']
   sudo: yes
   apt:
     name=apt-transport-https
     state=latest
 
 - name: add datadog apt key
-  tags: ['datadog']
   sudo: yes
   apt_key:
     id=C7A7DA52
     keyserver=keyserver.ubuntu.com
     state=present
 
 - name: add datadog repository
-  tags: ['datadog']
   sudo: yes
   apt_repository:
     repo='deb http://apt.datadoghq.com/ stable main'
     state=present
     update_cache=yes
 
+- name: install dd agent
+  sudo: yes
+  apt:
+    name=datadog-agent
+    state=latest
+
+- name: add docker to dd-agent group
+  sudo: yes
+  when: no_dd_monitoring != true
+  user:
+    name=dd-agent
+    groups=docker
+
+- name: create dd configdir
+  sudo: yes
+  file:
+    path=/etc/dd-agent/conf.d
+    owner=root
+    mode=0555
+    state=directory
+
 - name: Create main Datadog agent configuration file
-  tags: ['datadog']
   sudo: yes
   template:
     src=datadog.conf.j2
     dest=/etc/dd-agent/datadog.conf
   notify: restart datadog-agent
 
 - name: install network checks for dock services
-  tags: ['datadog']
   sudo: yes
   when: docker_config == "docks"
   template:
     src=tcp_check.yaml.j2
     dest=/etc/dd-agent/conf.d/tcp_check.yaml
   notify: restart datadog-agent
 
+- name: install docker integration
+  sudo: yes
+  when: no_dd_monitoring != true
+  template:
+    src=docker_daemon.yaml.j2
+    dest=/etc/dd-agent/conf.d/docker_daemon.yaml
+  notify: restart datadog-agent
+
+# assumes a config for {{ name }} exists
+- name: install datadog integrations
+  sudo: yes
+  when: docker_config != "docks"
+  template:
+    src="{{ name }}.yaml.j2"
+    dest="/etc/dd-agent/conf.d/{{ name }}.yaml"
+  notify: restart datadog-agent
+
 - name: force restart
-  tags: ['datadogg']
   command: echo restart datadog
   when: force_restart is defined
   notify: restart datadog-agent
diff --git a/ansible/roles/datadog/templates/docker_daemon.yaml.j2 b/ansible/roles/datadog/templates/docker_daemon.yaml.j2
@@ -0,0 +1,5 @@
+init_config:
+
+instances:
+  - url: "unix://var/run/docker.sock"
+    new_tag_names: true
diff --git a/ansible/roles/datadog/templates/mongo.yaml.j2 b/ansible/roles/datadog/templates/mongo.yaml.j2
@@ -0,0 +1,4 @@
+init_config:
+
+instances:
+  - server: mongodb://{{ datadog_mongodb_user }}:{{ datadog_mongodb_pwd }}@localhost:27017
diff --git a/ansible/roles/datadog/templates/rabbitmq.yaml.j2 b/ansible/roles/datadog/templates/rabbitmq.yaml.j2
@@ -0,0 +1,6 @@
+init_config:
+
+instances:
+  - {{ name }}_api_url: http://localhost:54320/api/
+    {{ name }}_user: {{ rabbit_username }}
+    {{ name }}_pass: {{ rabbit_password }}
diff --git a/ansible/workers.yml b/ansible/workers.yml
@@ -7,7 +7,7 @@
 - hosts: redis
 - hosts: swarm-manager
 
-- hosts: api
+- hosts: worker
   vars_files:
     - "group_vars/alpha-workers.yml"
   roles:
diff --git a/ssh/config b/ssh/config

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[defaults]`
	`2`	+# Required so `sudo: yes` does not lose the environment variables, which hold the ssh-agent socket
	`3`	`+sudo_flags=-HE`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@ docker_path: "/bin/docker"`
`13`	`13`	`dockerfile_pre_install_commands: [`
`14`	`14`	`"curl -L https://get.docker.com/builds/Linux/x86_64/docker-1.6.2 -o {{ docker_path }}",`
`15`	`15`	`"chmod a+x {{ docker_path }}",`
`16`		`- "curl -L https://github.com/weaveworks/weave/releases/download/v1.3.1/weave -o {{ weave_path }}",`
	`16`	`+ "curl -L https://github.com/weaveworks/weave/releases/download/v1.4.1/weave -o {{ weave_path }}",`
`17`	`17`	`"chmod a+x {{ weave_path }}"`
`18`	`18`	`]`
`19`	`19`