Skip to content

Commit 515b490

Browse files
author
Bryan Kendall
committed
add cloudwatch metrics as cronfile
1 parent b452cf8 commit 515b490

4 files changed

Lines changed: 254 additions & 0 deletions

File tree

‎ansible/gamma-hosts/variables‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,11 @@ registry_s3_region=us-east-1
7474
aws_access_key_id=AKIAJ3RCYU6FCULAJP2Q
7575
aws_secret_access_key=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
7676

77+
[swarm-manager:vars]
78+
aws_access_key=AKIAIB5W3E6HR6Q52HEQ
79+
aws_secret_key=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
80+
environment_name=gamma
81+
7782
[vault:vars]
7883
vault_hello_runnable_github_token=88ddc423c2312d02a8bbcaad76dd4c374a30e4af
7984
vault_aws_access_key_id=AKIAJ7R4UIM45KH2WGWQ

‎ansible/get-info.js‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
var AWS_ACCESS_KEY = process.env.AWS_ACCESS_KEY
2+
var AWS_SECRET_KEY = process.env.AWS_SECRET_KEY
3+
var ENVIRONMENT = process.env.ENVIRONMENT
4+
5+
var assign = require('101/assign')
6+
var AWS = require('aws-sdk')
7+
var Dockerode = require('dockerode')
8+
var find = require('101/find')
9+
var fs = require('fs')
10+
var join = require('path').join
11+
var Promise = require('bluebird')
12+
var Swarmerode = require('swarmerode')
13+
14+
AWS.config.update({
15+
accessKeyId: AWS_ACCESS_KEY,
16+
secretAccessKey: AWS_SECRET_KEY,
17+
region: 'us-west-2'
18+
})
19+
var ec2 = new AWS.EC2()
20+
var cloudwatch = new AWS.CloudWatch()
21+
22+
function promiseWhile (condition, action) {
23+
function loop (data) {
24+
if (condition(data)) { return Promise.resolve(data) }
25+
return action(data).then(loop)
26+
}
27+
return loop
28+
}
29+
30+
Dockerode = Swarmerode(Dockerode)
31+
32+
var certs = {}
33+
try {
34+
// DOCKER_CERT_PATH is docker's default thing it checks - may as well use it
35+
var certPath = process.env.DOCKER_CERT_PATH
36+
certs.ca = fs.readFileSync(join(certPath, '/ca.pem'))
37+
certs.cert = fs.readFileSync(join(certPath, '/cert.pem'))
38+
certs.key = fs.readFileSync(join(certPath, '/key.pem'))
39+
} catch (e) {
40+
console.error({ err: e }, 'cannot load certificates for docker!!')
41+
// use all or none - so reset certs here
42+
certs = {}
43+
}
44+
45+
var docker = new Dockerode(assign({
46+
host: process.env.SWARM_HOSTNAME,
47+
port: process.env.SWARM_PORT,
48+
timeout: 2 * 60 * 1000
49+
}, certs))
50+
51+
var UNITS = {
52+
'B': 'Bytes',
53+
'KiB': 'Kilobytes',
54+
'MiB': 'Megabytes',
55+
'GiB': 'Gigabytes'
56+
}
57+
58+
var FACTOR = {
59+
Bytes: 1000 * 1000 * 1000,
60+
Kilobytes: 1000 * 1000,
61+
Megabytes: 1000,
62+
Gigabytes: 1
63+
}
64+
65+
var FILTER_PARAMS = {
66+
Filters: [
67+
{ Name: 'tag:role', Values: ['dock'] },
68+
{
69+
Name: 'instance.group-name',
70+
Values: [ `${ENVIRONMENT}-dock` ]
71+
}
72+
]
73+
}
74+
75+
Promise.props({
76+
docks: getDocks(),
77+
swarmHosts: getSwarmInfo()
78+
})
79+
.then(function (data) {
80+
return Promise.each(data.swarmHosts, function (h) {
81+
var d = find(data.docks, function (dock) {
82+
return dock.PrivateIpAddress === h.Host
83+
})
84+
if (!d) {
85+
console.error('could not find match:', h.host)
86+
return
87+
}
88+
var org = find(d.Tags, function (t) { return t.Key === 'org' })
89+
if (!org) {
90+
console.error('could not find org tag')
91+
return
92+
}
93+
var orgID = org.Value
94+
var postData = {
95+
Namespace: 'Runnable/Swarm',
96+
MetricData: [
97+
{
98+
MetricName: 'Swarm Reserved Memory',
99+
Dimensions: [{
100+
Name: 'InstanceId',
101+
Value: d.InstanceId
102+
}, {
103+
Name: 'AutoScalingGroupName',
104+
Value: `asg-production-${ENVIRONMENT}-${orgID}`
105+
}],
106+
Value: h.Value,
107+
Unit: h.Unit
108+
}, {
109+
MetricName: 'Swarm Reserved Memory',
110+
Dimensions: [{
111+
Name: 'AutoScalingGroupName',
112+
Value: `asg-production-${ENVIRONMENT}-${orgID}`
113+
}],
114+
Value: h.Value,
115+
Unit: h.Unit
116+
}
117+
]
118+
}
119+
console.log(JSON.stringify(postData))
120+
return Promise.fromCallback(function (cb) {
121+
cloudwatch.putMetricData(postData, cb)
122+
})
123+
.then(function () { console.log('posted') })
124+
})
125+
})
126+
.catch(function (err) {
127+
console.error('ERROR:', err.stack || err.message || err)
128+
})
129+
130+
function getDocks () {
131+
return Promise.resolve({ instances: [] })
132+
.then(promiseWhile(
133+
function (data) { return data.done },
134+
function (data) {
135+
const opts = assign({}, FILTER_PARAMS)
136+
if (data.NextToken) { opts.NextToken = data.NextToken }
137+
return Promise.fromCallback(function (cb) {
138+
ec2.describeInstances(opts, cb)
139+
})
140+
.then(function (awsData) {
141+
awsData.Reservations.forEach(function (r) {
142+
r.Instances.forEach(function (i) {
143+
data.instances.push(i)
144+
})
145+
})
146+
data.NextToken = awsData.NextToken
147+
data.done = !awsData.NextToken
148+
return data
149+
})
150+
}
151+
))
152+
.then(function (data) { return data.instances })
153+
}
154+
155+
function getSwarmInfo () {
156+
return Promise.fromCallback(function (cb) {
157+
docker.swarmInfo(cb)
158+
})
159+
.then(function (info) {
160+
return Object.keys(info.parsedSystemStatus.ParsedNodes).map(function (key) {
161+
var node = info.parsedSystemStatus.ParsedNodes[key]
162+
var usedMemory = node.ReservedMem.split('/').shift().trim()
163+
var availableMemory = node.ReservedMem.split('/').pop().trim()
164+
var usedMemoryValue = parseFloat(usedMemory.split(' ').shift())
165+
var usedMemoryUnits = UNITS[usedMemory.split(' ').pop()]
166+
var availableMemoryValue = parseFloat(availableMemory.split(' ').shift())
167+
var availableMemoryUnits = UNITS[availableMemory.split(' ').pop()]
168+
169+
var usedMemoryGiB = usedMemoryValue / FACTOR[usedMemoryUnits]
170+
var availableMemoryGiB = availableMemoryValue / FACTOR[availableMemoryUnits]
171+
172+
var percentage = (usedMemoryGiB / availableMemoryGiB) * 100
173+
174+
return {
175+
Host: node.Host.split(':').shift(),
176+
Value: percentage,
177+
Unit: 'Percent'
178+
}
179+
})
180+
})
181+
}

‎ansible/group_vars/alpha-swarm-manager-metrics.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
name: swarm-cloudwatch-reporter
3+
4+
container_image: node
5+
container_tag: 4
6+
7+
log_driver: json-file
8+
9+
container_run_opts: >-
10+
--log-driver={{ log_driver }}
11+
-e AWS_ACCESS_KEY={{ aws_access_key }}
12+
-e AWS_SECRET_KEY={{ aws_secret_key }}
13+
-e DOCKER_CERT_PATH=/opt/ssl/docker/swarm-manager
14+
-e ENVIRONMENT={{ environment_name }}
15+
-e SWARM_HOSTNAME={{ ansible_default_ipv4.address }}
16+
-e SWARM_PORT={{ swarm_master_port }}
17+
-v /opt/runnable/get-info.js:/get-info.js:ro
18+
-v /opt/ssl/docker/swarm-manager:/opt/ssl/docker/swarm-manager:ro
19+
-v /var/log:/var/log
20+
21+
commands_to_run: >-
22+
npm install 101 swarmerode dockerode bluebird aws-sdk &&
23+
node get-info.js
24+
container_run_args: bash -c "{{ commands_to_run }}" >> /var/log/{{ name }}.log 2>&1

‎ansible/swarm-cloudwatch-reporter.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
---
2+
- hosts: swarm-manager
3+
vars_files:
4+
- group_vars/alpha-swarm-manager-metrics.yml
5+
tasks:
6+
- name: make runnable folder
7+
become: yes
8+
file:
9+
dest: /opt/runnable/
10+
state: directory
11+
owner: ubuntu
12+
group: ubuntu
13+
mode: 0775
14+
15+
- name: put script in place
16+
template:
17+
src: get-info.js
18+
dest: /opt/runnable/get-info.js
19+
owner: ubuntu
20+
group: ubuntu
21+
mode: 0444
22+
23+
- name: run container
24+
when: test_run is defined
25+
become: yes
26+
shell: >-
27+
docker run
28+
{{ container_run_opts }}
29+
{{ container_image }}:{{ container_tag }}
30+
{{ container_run_args }}
31+
32+
- name: put script into cron
33+
become: yes
34+
cron:
35+
name: swarm-cloudwatch-reporter
36+
cron_file: 10-swarm-cloudwatch
37+
user: root
38+
state: present
39+
job: >-
40+
sudo docker run
41+
--rm
42+
{{ container_run_opts }}
43+
{{ container_image }}:{{ container_tag }}
44+
{{ container_run_args }}

0 commit comments

Comments
 (0)