better prom for kube

Anandkumar Patel · Anandkumar Patel · commit 013322273af9 · 2017-05-26T10:47:53.000-07:00
diff --git a/ansible/delta-hosts/variables b/ansible/delta-hosts/variables
@@ -140,6 +140,7 @@ pheidi_runnabot_tokens=d22a5ec75f66a0f472f3d856eb01df2321115e0b,33d8accaa7afeace
 
 [prometheus:vars]
 pager_duty_key=ky6EFGL7Zh-xH8x1qEJV
+volume_id=vol-0dc9ca42481538a30
 
 [sauron:vars]
 sauron_rollbar_key=83157ae2d50d4b6398e404c0b9978d26
diff --git a/ansible/group_vars/alpha-prometheus.yml b/ansible/group_vars/alpha-prometheus.yml
@@ -6,15 +6,20 @@ hosted_ports: ["{{ prometheus_port }}"]
 
 memory_hard_limit: 15G
 
+storage_size_gb: 100
+
 volume_mounts:
- - name: "{{ name }}-conf"
-   path: /prometheus
-   kind: configMap
+  - name: "{{ name }}-conf"
+    path: /prometheus
+    kind: configMap
+  - name: "{{ name }}-db-claim"
+    path: /data
+    kind: persistent
 
 container_run_args: >
   -alertmanager.url "{{ prometheus_alert_url }}"
   -config.file=/prometheus/prometheus.yml
-  -storage.local.path "data"
+  -storage.local.path "/data"
   -storage.local.retention 168h0m0s
   -web.external-url http://localhost
   -web.listen-address ":{{ prometheus_port }}"
diff --git a/ansible/prometheus.yml b/ansible/prometheus.yml
@@ -7,5 +7,6 @@
 
   - role: prometheus
 
+  - role: database
   - role: k8-deployment
   - role: k8-service
diff --git a/ansible/roles/database/tasks/main.yml b/ansible/roles/database/tasks/main.yml
@@ -10,10 +10,10 @@
 
 - name: create volume yaml
   template:
-    dest: "{{ volumes_path }}/{{ name }}"
+    dest: "{{ volumes_path }}/{{ name }}.yml"
     src: volume.yml
 
 - name: create volume claim yaml
   template:
-    dest: "{{ volumes_path }}/{{ name }}-claim"
+    dest: "{{ volumes_path }}/{{ name }}-claim.yml"
     src: volume-claim.yml
diff --git a/ansible/roles/prometheus/files/alerts.conf b/ansible/roles/prometheus/files/alerts.conf
@@ -6,8 +6,8 @@ ALERT HookDockUnresponsive
     type = "unresponsive"
   }
   ANNOTATIONS {
-    summary = "({{ $labels.env }}) Dock unresponsive host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",
-    description = "(hook) Dock unresponsive host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}"
+    summary = "({{ $labels.env }}) Dock unresponsive host={{ $labels.hostIp }} labels={{ $labels }}",
+    description = "(hook) Dock unresponsive host={{ $labels.hostIp }} labels={{ $labels }}"
   }
 
 ALERT DockUnresponsive
@@ -17,8 +17,8 @@ ALERT DockUnresponsive
     reportTo = "pagerduty"
   }
   ANNOTATIONS {
-    summary = "({{ $labels.env }}) Dock unresponsive host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",
-    description = "Dock unresponsive host={{ $labels.hostIp }} org={{ $labels.githubOrgId }"
+    summary = "({{ $labels.env }}) Dock unresponsive host={{ $labels.hostIp }} labels={{ $labels }}",
+    description = "Dock unresponsive host={{ $labels.hostIp }} labels={{ $labels }"
   }
 
 ALERT HookDockDockerDiskFull
@@ -29,8 +29,8 @@ ALERT HookDockDockerDiskFull
     type = "disk_filled"
   }
   ANNOTATIONS {
-    summary = "({{ $labels.env }}) Dock /docker disk 70% host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",
-    description = "(hook) Dock /docker disk 70% host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}"
+    summary = "({{ $labels.env }}) Dock /docker disk 70% host={{ $labels.hostIp }} labels={{ $labels }}",
+    description = "(hook) Dock /docker disk 70% host={{ $labels.hostIp }} labels={{ $labels }}"
   }
 
 ALERT DockDockerDiskFull
@@ -40,7 +40,7 @@ ALERT DockDockerDiskFull
     reportTo = "pagerduty"
   }
   ANNOTATIONS {
-    summary = "({{ $labels.env }}) Dock /docker disk 90% host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",
+    summary = "({{ $labels.env }}) Dock /docker disk 90% host={{ $labels.hostIp }} labels={{ $labels }}",
     description = "Playbook here: https://github.com/CodeNow/devops-scripts/wiki/server-out-of-disk"
   }
 
@@ -51,7 +51,7 @@ ALERT DockRootDiskFull
     reportTo = "pagerduty"
   }
   ANNOTATIONS {
-    summary = "({{ $labels.env }}) Dock root disk 90% host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",
+    summary = "({{ $labels.env }}) Dock root disk 90% host={{ $labels.hostIp }} labels={{ $labels }}",
     description = "Playbook here: https://github.com/CodeNow/devops-scripts/wiki/server-out-of-disk"
   }
 
@@ -63,8 +63,8 @@ ALERT HookDockOutOfRam
     type = "memory_exhausted"
   }
   ANNOTATIONS {
-    summary = "({{ $labels.env }}) Dock out of ram host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",
-    description = "(hook) Dock out of ram host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}"
+    summary = "({{ $labels.env }}) Dock out of ram host={{ $labels.hostIp }} labels={{ $labels }}",
+    description = "(hook) Dock out of ram host={{ $labels.hostIp }} labels={{ $labels }}"
   }
 
 ALERT DockOutOfRam
@@ -74,8 +74,8 @@ ALERT DockOutOfRam
     reportTo = "pagerduty"
   }
   ANNOTATIONS {
-    summary = "({{ $labels.env }}) Dock out of ram host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",
-    description = "unhealthy dock {{ $labels.hostIp } using dock-cli and message slack #customer channel with org={{ $labels.githubOrgId }}"
+    summary = "({{ $labels.env }}) Dock out of ram host={{ $labels.hostIp }} labels={{ $labels }}",
+    description = "unhealthy dock {{ $labels.hostIp } using dock-cli and message slack #customer channel with labels={{ $labels }}"
   }
 
 ALERT DockHighLoad
@@ -85,6 +85,6 @@ ALERT DockHighLoad
     reportTo = "pagerduty"
   }
   ANNOTATIONS {
-    summary = "({{ $labels.env }}) Dock is experiencing high load host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",
+    summary = "({{ $labels.env }}) Dock is experiencing high load host={{ $labels.hostIp }} labels={{ $labels }}",
     description = "ssh {{ $labels.hostIp }} into dock make sure it is responsive, if it is not, unhealthy. `docks unhealthy -e delta {{ $labels.hostIp }}`"
   }

Original file line number	Diff line number	Diff line change
`@@ -6,8 +6,8 @@ ALERT HookDockUnresponsive`
`6`	`6`	`type = "unresponsive"`
`7`	`7`	`}`
`8`	`8`	`ANNOTATIONS {`
`9`		`- summary = "({{ $labels.env }}) Dock unresponsive host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",`
`10`		`- description = "(hook) Dock unresponsive host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}"`
	`9`	`+ summary = "({{ $labels.env }}) Dock unresponsive host={{ $labels.hostIp }} labels={{ $labels }}",`
	`10`	`+ description = "(hook) Dock unresponsive host={{ $labels.hostIp }} labels={{ $labels }}"`
`11`	`11`	`}`
`12`	`12`
`13`	`13`	`ALERT DockUnresponsive`
`@@ -17,8 +17,8 @@ ALERT DockUnresponsive`
`17`	`17`	`reportTo = "pagerduty"`
`18`	`18`	`}`
`19`	`19`	`ANNOTATIONS {`
`20`		`- summary = "({{ $labels.env }}) Dock unresponsive host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",`
`21`		`- description = "Dock unresponsive host={{ $labels.hostIp }} org={{ $labels.githubOrgId }"`
	`20`	`+ summary = "({{ $labels.env }}) Dock unresponsive host={{ $labels.hostIp }} labels={{ $labels }}",`
	`21`	`+ description = "Dock unresponsive host={{ $labels.hostIp }} labels={{ $labels }"`
`22`	`22`	`}`
`23`	`23`
`24`	`24`	`ALERT HookDockDockerDiskFull`
`@@ -29,8 +29,8 @@ ALERT HookDockDockerDiskFull`
`29`	`29`	`type = "disk_filled"`
`30`	`30`	`}`
`31`	`31`	`ANNOTATIONS {`
`32`		`- summary = "({{ $labels.env }}) Dock /docker disk 70% host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",`
`33`		`- description = "(hook) Dock /docker disk 70% host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}"`
	`32`	`+ summary = "({{ $labels.env }}) Dock /docker disk 70% host={{ $labels.hostIp }} labels={{ $labels }}",`
	`33`	`+ description = "(hook) Dock /docker disk 70% host={{ $labels.hostIp }} labels={{ $labels }}"`
`34`	`34`	`}`
`35`	`35`
`36`	`36`	`ALERT DockDockerDiskFull`
`@@ -40,7 +40,7 @@ ALERT DockDockerDiskFull`
`40`	`40`	`reportTo = "pagerduty"`
`41`	`41`	`}`
`42`	`42`	`ANNOTATIONS {`
`43`		`- summary = "({{ $labels.env }}) Dock /docker disk 90% host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",`
	`43`	`+ summary = "({{ $labels.env }}) Dock /docker disk 90% host={{ $labels.hostIp }} labels={{ $labels }}",`
`44`	`44`	`description = "Playbook here: https://github.com/CodeNow/devops-scripts/wiki/server-out-of-disk"`
`45`	`45`	`}`
`46`	`46`
`@@ -51,7 +51,7 @@ ALERT DockRootDiskFull`
`51`	`51`	`reportTo = "pagerduty"`
`52`	`52`	`}`
`53`	`53`	`ANNOTATIONS {`
`54`		`- summary = "({{ $labels.env }}) Dock root disk 90% host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",`
	`54`	`+ summary = "({{ $labels.env }}) Dock root disk 90% host={{ $labels.hostIp }} labels={{ $labels }}",`
`55`	`55`	`description = "Playbook here: https://github.com/CodeNow/devops-scripts/wiki/server-out-of-disk"`
`56`	`56`	`}`
`57`	`57`
`@@ -63,8 +63,8 @@ ALERT HookDockOutOfRam`
`63`	`63`	`type = "memory_exhausted"`
`64`	`64`	`}`
`65`	`65`	`ANNOTATIONS {`
`66`		`- summary = "({{ $labels.env }}) Dock out of ram host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",`
`67`		`- description = "(hook) Dock out of ram host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}"`
	`66`	`+ summary = "({{ $labels.env }}) Dock out of ram host={{ $labels.hostIp }} labels={{ $labels }}",`
	`67`	`+ description = "(hook) Dock out of ram host={{ $labels.hostIp }} labels={{ $labels }}"`
`68`	`68`	`}`
`69`	`69`
`70`	`70`	`ALERT DockOutOfRam`
`@@ -74,8 +74,8 @@ ALERT DockOutOfRam`
`74`	`74`	`reportTo = "pagerduty"`
`75`	`75`	`}`
`76`	`76`	`ANNOTATIONS {`
`77`		`- summary = "({{ $labels.env }}) Dock out of ram host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",`
`78`		`- description = "unhealthy dock {{ $labels.hostIp } using dock-cli and message slack #customer channel with org={{ $labels.githubOrgId }}"`
	`77`	`+ summary = "({{ $labels.env }}) Dock out of ram host={{ $labels.hostIp }} labels={{ $labels }}",`
	`78`	`+ description = "unhealthy dock {{ $labels.hostIp } using dock-cli and message slack #customer channel with labels={{ $labels }}"`
`79`	`79`	`}`
`80`	`80`
`81`	`81`	`ALERT DockHighLoad`
`@@ -85,6 +85,6 @@ ALERT DockHighLoad`
`85`	`85`	`reportTo = "pagerduty"`
`86`	`86`	`}`
`87`	`87`	`ANNOTATIONS {`
`88`		`- summary = "({{ $labels.env }}) Dock is experiencing high load host={{ $labels.hostIp }} org={{ $labels.githubOrgId }}",`
	`88`	`+ summary = "({{ $labels.env }}) Dock is experiencing high load host={{ $labels.hostIp }} labels={{ $labels }}",`
`89`	`89`	description = "ssh {{ $labels.hostIp }} into dock make sure it is responsive, if it is not, unhealthy. `docks unhealthy -e delta {{ $labels.hostIp }}`"
`90`	`90`	`}`