Merge remote-tracking branch 'origin/master' into SAN-3043-cache

cflynn07 · cflynn07 · commit b39744803690 · 2016-01-11T16:46:26.000-08:00
* origin/master: (29 commits)
  Added link to list
  Added detention to list
  that's the ticket!
  (oops)
  Delta.
  wrong IP for gamma-api-worker
  move repo to repo folder so we do not conflit when adding dockerfile
  remove depth
  Revert "Revert "modify building for faster deploys""
  change to fix deploy workers
  changed worker in gamma-hosts/hosts from gamma-worker to gamma-api-worker
  updated config file update ec2 instance and update gamma host with different worker
  change comment
  dummy dumb dumb
  last minute fixes
  missing ""
  Wiping the Redis check, as it does not work.
  tag entire role
  add metis to gamma
  update weave
  ...
diff --git a/README.md b/README.md
@@ -105,4 +105,8 @@ It is the custom at Runnable to play a song to the entire team when deploying. F
 
 [Sauron: Sauron theme song from LOTR](https://www.youtube.com/watch?v=V_rk9VBrXMY)
 
+[Detention: Unbreakable Kimmy Schmidt](https://youtu.be/CV9xF8CjhJk?t=21s)
+
+[Link: Zelda Main Theme Song](https://www.youtube.com/watch?v=cGufy1PAeTU)
+
 **IMPORTANT:** Make sure the play the song loud and proud when deploying!
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
@@ -0,0 +1,3 @@
+[defaults]
+# Required so `sudo: yes` does not lose the environment variables, which hold the ssh-agent socket
+sudo_flags=-HE
diff --git a/ansible/delta-hosts/hosts b/ansible/delta-hosts/hosts
@@ -29,7 +29,7 @@ delta-web
 delta-services
 
 [worker]
-delta-api
+delta-api-worker
 
 [docks]
 
@@ -95,6 +95,7 @@ bastion
 hipache
 mongodb
 api
+worker
 web
 redis
 redis-slave
diff --git a/ansible/delta-hosts/variables b/ansible/delta-hosts/variables
@@ -63,8 +63,9 @@ domain=runnable.io
 mongo_port=27017
 new_relic_license_key=338516e0826451c297d44dc60aeaf0a0ca4bfead
 node_env=production-delta
-pg_host=delta-infrastructure-db.cnksgdqarobf.us-west-2.rds.amazonaws.com
+pg_host=delta-infradb.cnksgdqarobf.us-west-2.rds.amazonaws.com
 pg_pass=59a5524e-a772-11e5-bedc-1bdc0db458b3
+pg_user=astral
 rabbit_password=wKK7g7NWKpQXEeSzyWB7mIpxZIL8H2mDSf3Q6czR3Vk
 rabbit_username=o2mdLh9N9Ke2GzhoK8xsruYPhIQFN7iEL44dQJoq7OM
 registry_host=10.8.4.126
diff --git a/ansible/docks.yml b/ansible/docks.yml
@@ -10,7 +10,7 @@
   roles:
     - { role: docker, tags: "docker" }
     - { role: iptables, tags: "iptables, security" }
-    - { role: datadog }
+    - { role: datadog, tags: "datadog" }
 
 - include: krain.yml
 - include: filibuster.yml
diff --git a/ansible/gamma-hosts/hosts b/ansible/gamma-hosts/hosts
@@ -29,7 +29,7 @@ gamma-web
 gamma-services
 
 [worker]
-gamma-api
+gamma-api-worker
 
 [eru]
 gamma-services
@@ -85,13 +85,17 @@ gamma-registry
 [swarm-manager]
 gamma-services
 
+[metis]
+gamma-services
+
 [docks]
 
 [gamma:children]
 bastion
 hipache
 mongodb
 api
+worker
 web
 redis
 redis-slave
@@ -108,6 +112,7 @@ eru
 sauron
 shiva
 swarm-manager
+metis
 
 [ec2]
 127.0.0.1
diff --git a/ansible/gamma-hosts/variables b/ansible/gamma-hosts/variables
@@ -58,6 +58,8 @@ vault_aws_region=us-east-1
 ansible_ssh_private_key_file=~/.ssh/gamma.pem
 datadog_host_address=10.4.6.251
 datadog_tags=env:gamma
+datadog_mongodb_user=datadog
+datadog_mongodb_pwd=sqa3WBgkCgZsFZuex0kBNahZ
 domain=runnable-gamma.com
 mongo_port=27017
 new_relic_license_key=338516e0826451c297d44dc60aeaf0a0ca4bfead
@@ -70,6 +72,7 @@ registry_host=10.4.4.82
 swarm_token=d363b783f03a845a2c82b081bfe8443e
 user_content_domain=runnable.ninja
 api_hello_runnable_github_token=88ddc423c2312d02a8bbcaad76dd4c374a30e4af
+no_dd_monitoring=false
 
 [ec2:vars]
 env=gamma
diff --git a/ansible/group_vars/alpha-mongo.yml b/ansible/group_vars/alpha-mongo.yml
@@ -7,3 +7,6 @@ db_path: "/opt/mongodb/db"
 container_image: mongo
 container_tag: "latest"
 container_run_opts: "-d --name mongodb -p 27017:27017 -v {{ db_path }}:/data/db:rw"
+
+# do not monitor docker-daemon for mongo, as there is none
+no_dd_monitoring: true
diff --git a/ansible/group_vars/alpha-rabbitmq.yml b/ansible/group_vars/alpha-rabbitmq.yml
@@ -10,3 +10,5 @@ container_tag: "3.5.3-management"
 container_run_opts: "-d --name prod-rabbit -v {{ db_path }}:/var/lib/rabbitmq:rw -v {{ config_path }}/rabbitmq.config:/etc/rabbitmq/rabbitmq.config:ro -p 54320:15672 -p 54321:5672"
 
 hosted_ports: [ 5672, 15672 ]
+# monitor the docker-daemon
+no_dd_monitoring: false
diff --git a/ansible/group_vars/alpha-sauron.yml b/ansible/group_vars/alpha-sauron.yml
@@ -13,7 +13,7 @@ docker_path: "/bin/docker"
 dockerfile_pre_install_commands: [
   "curl -L https://get.docker.com/builds/Linux/x86_64/docker-1.6.2 -o {{ docker_path }}",
   "chmod a+x {{ docker_path }}",
-  "curl -L https://github.com/weaveworks/weave/releases/download/v1.3.1/weave -o {{ weave_path }}",
+  "curl -L https://github.com/weaveworks/weave/releases/download/v1.4.1/weave -o {{ weave_path }}",
   "chmod a+x {{ weave_path }}"
 ]
 
diff --git a/ansible/mongo.yml b/ansible/mongo.yml
@@ -6,3 +6,4 @@
   - { role: notify, tags: "notify" }
   - { role: database }
   - { role: container_kill_start, tags: "deploy" }
+  - { role: datadog, tags: "datadog" }
diff --git a/ansible/rabbitmq.yml b/ansible/rabbitmq.yml
@@ -7,3 +7,4 @@
   - { role: database }
   - { role: rabbitmq }
   - { role: container_kill_start, tags: "deploy" }
+  - { role: datadog, tags: "datadog" }
diff --git a/ansible/roles/builder/tasks/main.yml b/ansible/roles/builder/tasks/main.yml
@@ -6,6 +6,16 @@
     path="{{ build_dir }}/{{ name }}"
     state=directory
 
+- name: pull the git repository
+  sudo: yes
+  git:
+    repo="{{ repo }}"
+    dest="{{ build_dir }}/{{ name }}/repo"
+    version="{{ git_branch }}"
+    update=yes
+    accept_hostkey=True
+    force=yes
+
 - name: copy dockerfile to build folder
   sudo: yes
   template:
diff --git a/ansible/roles/builder/templates/basic_node/Dockerfile b/ansible/roles/builder/templates/basic_node/Dockerfile
@@ -15,18 +15,22 @@ ENV {{ env }}
 # setup node and npm versions
 RUN n {{ node_version }} && npm install -g npm@{{ npm_version }}
 
-# Download Repo
-RUN git clone -b {{ git_branch }} --single-branch {{ repo }} /{{ name }}
-
-WORKDIR /{{ name }}
 {% if dockerfile_pre_install_commands is defined %}
 {% for command in dockerfile_pre_install_commands %}
 RUN {{ command }}
 {% endfor %}
 {% endif %}
 
+# Add package.json from the current build context (`.` is the repo) first
+ADD ./repo/package.json /{{ name }}/package.json
+
+# install, should will skip if no package.json change
+WORKDIR /{{ name }}
 RUN npm install --production
 
+# move the current build context (`.` is the repo) to /{{ name }}
+ADD ./repo /{{ name }}
+
 {% if dockerfile_post_install_commands is defined %}
 {% for command in dockerfile_post_install_commands %}
 RUN {{ command }}
@@ -35,4 +39,3 @@ RUN {{ command }}
 
 # Define default command.
 CMD ulimit -c unlimited && /usr/local/bin/npm {{ npm_start_command | default('start') }} > /var/log/{{ name }}.log 2>&1
-
diff --git a/ansible/roles/datadog/tasks/main.yml b/ansible/roles/datadog/tasks/main.yml
@@ -1,46 +1,78 @@
 ---
 - name: add https transport
-  tags: ['datadog']
   sudo: yes
   apt:
     name=apt-transport-https
     state=latest
 
 - name: add datadog apt key
-  tags: ['datadog']
   sudo: yes
   apt_key:
     id=C7A7DA52
     keyserver=keyserver.ubuntu.com
     state=present
 
 - name: add datadog repository
-  tags: ['datadog']
   sudo: yes
   apt_repository:
     repo='deb http://apt.datadoghq.com/ stable main'
     state=present
     update_cache=yes
 
+- name: install dd agent
+  sudo: yes
+  apt:
+    name=datadog-agent
+    state=latest
+
+- name: add docker to dd-agent group
+  sudo: yes
+  when: no_dd_monitoring != true
+  user:
+    name=dd-agent
+    groups=docker
+
+- name: create dd configdir
+  sudo: yes
+  file:
+    path=/etc/dd-agent/conf.d
+    owner=root
+    mode=0555
+    state=directory
+
 - name: Create main Datadog agent configuration file
-  tags: ['datadog']
   sudo: yes
   template:
     src=datadog.conf.j2
     dest=/etc/dd-agent/datadog.conf
   notify: restart datadog-agent
 
 - name: install network checks for dock services
-  tags: ['datadog']
   sudo: yes
   when: docker_config == "docks"
   template:
     src=tcp_check.yaml.j2
     dest=/etc/dd-agent/conf.d/tcp_check.yaml
   notify: restart datadog-agent
 
+- name: install docker integration
+  sudo: yes
+  when: no_dd_monitoring != true
+  template:
+    src=docker_daemon.yaml.j2
+    dest=/etc/dd-agent/conf.d/docker_daemon.yaml
+  notify: restart datadog-agent
+
+# assumes a config for {{ name }} exists
+- name: install datadog integrations
+  sudo: yes
+  when: docker_config != "docks"
+  template:
+    src="{{ name }}.yaml.j2"
+    dest="/etc/dd-agent/conf.d/{{ name }}.yaml"
+  notify: restart datadog-agent
+
 - name: force restart
-  tags: ['datadogg']
   command: echo restart datadog
   when: force_restart is defined
   notify: restart datadog-agent
diff --git a/ansible/roles/datadog/templates/docker_daemon.yaml.j2 b/ansible/roles/datadog/templates/docker_daemon.yaml.j2
@@ -0,0 +1,5 @@
+init_config:
+
+instances:
+  - url: "unix://var/run/docker.sock"
+    new_tag_names: true
diff --git a/ansible/roles/datadog/templates/mongo.yaml.j2 b/ansible/roles/datadog/templates/mongo.yaml.j2
@@ -0,0 +1,4 @@
+init_config:
+
+instances:
+  - server: mongodb://{{ datadog_mongodb_user }}:{{ datadog_mongodb_pwd }}@localhost:27017
diff --git a/ansible/roles/datadog/templates/rabbitmq.yaml.j2 b/ansible/roles/datadog/templates/rabbitmq.yaml.j2
@@ -0,0 +1,6 @@
+init_config:
+
+instances:
+  - {{ name }}_api_url: http://localhost:54320/api/
+    {{ name }}_user: {{ rabbit_username }}
+    {{ name }}_pass: {{ rabbit_password }}
diff --git a/ansible/workers.yml b/ansible/workers.yml
@@ -7,7 +7,7 @@
 - hosts: redis
 - hosts: swarm-manager
 
-- hosts: api
+- hosts: worker
   vars_files:
     - "group_vars/alpha-workers.yml"
   roles:
diff --git a/ssh/config b/ssh/config
@@ -82,6 +82,12 @@ Host gamma-api
 Host gamma-api-us-west-2a
   ProxyCommand ssh -q ubuntu@gamma-bastion nc 10.4.7.170 22
 
+Host gamma-api-worker
+  ProxyCommand ssh -q ubuntu@gamma-bastion nc 10.4.4.212 22
+
+Host gamma-api-worker-us-west-2a
+  ProxyCommand ssh -q ubuntu@gamma-bastion nc 10.4.4.212 22
+
 ## Gamma MongoDB
 Host gamma-mongo-a
   ProxyCommand ssh -q ubuntu@gamma-bastion nc 10.4.7.56 22
@@ -164,6 +170,9 @@ Host delta-admin
 Host delta-api
   ProxyCommand ssh -q ubuntu@delta-bastion nc 10.8.5.212 22
 
+Host delta-api-worker
+  ProxyCommand ssh -q ubuntu@delta-bastion nc 10.8.6.132 22
+
 Host delta-services
   ProxyCommand ssh -q ubuntu@delta-bastion nc 10.8.5.63 22
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[defaults]`
	`2`	+# Required so `sudo: yes` does not lose the environment variables, which hold the ssh-agent socket
	`3`	`+sudo_flags=-HE`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@ docker_path: "/bin/docker"`
`13`	`13`	`dockerfile_pre_install_commands: [`
`14`	`14`	`"curl -L https://get.docker.com/builds/Linux/x86_64/docker-1.6.2 -o {{ docker_path }}",`
`15`	`15`	`"chmod a+x {{ docker_path }}",`
`16`		`- "curl -L https://github.com/weaveworks/weave/releases/download/v1.3.1/weave -o {{ weave_path }}",`
	`16`	`+ "curl -L https://github.com/weaveworks/weave/releases/download/v1.4.1/weave -o {{ weave_path }}",`
`17`	`17`	`"chmod a+x {{ weave_path }}"`
`18`	`18`	`]`
`19`	`19`