Merge branch 'master' into SAN-4042-log-cleanup

Author: Christopher M. Neill

ansible/group_vars/all.yml

@@ -32,7 +32,7 @@ loggly_username: ops
 ## cores and logs
 ##
 
-app_log_dir: /var/log
+app_log_dir: /var/log/runnable
 core_file_dir: /var/log/core
 
 ##

ansible/roles/bash_aliases/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+- name: copy bash aliases to ubuntu
+  tags: [ loggly, bash_aliases ] 
+  template:
+    src=dot_bash_aliases.sh.j2
+    dest=/home/ubuntu/.bash_aliases
+    owner=ubuntu
+    group=ubuntu
+    mode=0700
+
+- name: copy bash aliases to root
+  tags: [ loggly, bash_aliases ] 
+  become: true
+  template:
+    src=dot_bash_aliases_root.sh.j2
+    dest=/root/.bash_aliases
+    owner=root
+    group=root
+    mode=0700

ansible/roles/bash_aliases/templates/dot_bash_aliases.sh.j2

@@ -0,0 +1,19 @@
+# Follows the logfile for a given app_name interpolating the datetime string into the logpath (/var/log/runnable/YYYY/MM/DD/HH/<app_name>.log)
+# Usage: logtail <app_name>
+function logtail() {
+    local app_name="$1"
+    local datetime=`date +%Y/%m/%d/%H`
+    local app_log_dir="{{ app_log_dir }}"
+    local logfile="${app_log_dir}/${datetime}/${app_name}.log"
+    tail -f ${logfile} | bunyan
+}
+
+# Outputs contents of an npm start log for <app_name>, if it exists, into a pager for reading.
+# Usage: npmlog <app_name>
+function npmlog() {
+    local app_name="$1"
+    local app_log_dir="/var/log"
+    local logfile="${app_log_dir}/${app_name}.log"
+    less ${logfile}
+}
+

ansible/roles/bash_aliases/templates/dot_bash_aliases_root.sh.j2

@@ -0,0 +1,19 @@
+# Follows the logfile for a given app_name interpolating the datetime string into the logpath (/var/log/runnable/YYYY/MM/DD/HH/<app_name>.log)
+# Usage: logtail <app_name>
+function logtail() {
+    local app_name="$1"
+    local datetime=`date +%Y/%m/%d/%H`
+    local app_log_dir="{{ app_log_dir }}"
+    local logfile="${app_log_dir}/${datetime}/${app_name}.log"
+    tail -f ${logfile} | bunyan
+}
+
+# Outputs contents of an npm start log for <app_name>, if it exists, into a pager for reading.
+# Usage: npmlog <app_name>
+function npmlog() {
+    local app_name="$1"
+    local app_log_dir="/var/log"
+    local logfile="${app_log_dir}/${app_name}.log"
+    less ${logfile}
+}
+

…e/roles/loggly/templates/rsyslog.conf.j2 ansible/roles/loggly/files/rsyslog.confansible/roles/loggly/templates/rsyslog.conf.j2 renamed to ansible/roles/loggly/files/rsyslog.conf ansible/roles/loggly/templates/rsyslog.conf.j2 renamed to ansible/roles/loggly/files/rsyslog.conf

@@ -35,6 +35,11 @@ $KLogPermitNonKernelFacility on
 #
 $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
 
+# Runnable JSON logging format
+# Creates an "output formatter" template that accepts as input JSON and prints it out without any further processing ("raw JSON").
+# The formatting around the %msg% string is as such: start printing at the second character "2" until the end of the line "$" using the raw JSON format type.
+$template RunnableJSON,"%msg:2:$:jsonr%\n"
+
 # Filter duplicated messages
 $RepeatedMsgReduction on
 
@@ -43,11 +48,11 @@ $RepeatedMsgReduction on
 #
 $FileOwner syslog
 $FileGroup adm
-$FileCreateMode 0640
+$FileCreateMode 0644
 $DirCreateMode 0755
 $Umask 0022
 $PrivDropToUser syslog
-$PrivDropToGroup syslog
+$PrivDropToGroup adm
 
 #
 # Where to place spool and state files

ansible/roles/loggly/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: bash_aliases }

ansible/roles/loggly/tasks/main.yml

@@ -46,25 +46,6 @@
     owner=syslog
     group=syslog
 
-- name: create runnable bin directory
-  tags: loggly
-  become: true
-  file:
-    path=/opt/runnable/bin
-    state=directory
-    mode=0755
-    owner=ubuntu
-    group=ubuntu
-
-- name: copy rotate util script
-  tags: loggly
-  when: name is defined
-  become: true
-  template:
-    src=rotate-logs.sh.j2
-    dest=/opt/runnable/bin/rotate-{{ name }}-logs.sh
-    mode=0755
-
 - name: copy app config
   tags: loggly
   become: true
@@ -77,8 +58,8 @@
 - name: copy rsyslog config
   tags: loggly
   become: true
-  template:
-    src=rsyslog.conf.j2
+  copy:
+    src=rsyslog.conf
     dest=/etc/rsyslog.conf
     owner=syslog
     group=syslog
@@ -95,6 +76,17 @@
     path=/var/spool/rsyslog/stat-{{ name }}
     state=absent
 
+- name: ensure log path
+  tags: loggly
+  become: true
+  file:
+    path="{{ app_log_dir }}"
+    state=directory
+    owner=syslog
+    group=adm
+    mode=0755
+    recurse=yes
+
 - name: restart rsyslog
   tags: [ loggly, deploy ]
   become: true
@@ -141,4 +133,3 @@
   become: true
   script: archiveOldLogs.sh
   register: purge_out
-

ansible/roles/loggly/templates/21-output-syslog.conf.j2

@@ -1,14 +1,10 @@
 $WorkDirectory /var/spool/rsyslog
 
-# start log rotation via outchannel
-# outchannel definition
-$outchannel log_rotation_{{ name }},{{ app_log_dir }}/{{ name }}-daemon.log,524288000,/opt/runnable/bin/rotate-{{ name }}-logs.sh
-#  activate the channel and log everything to it
-if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then :omfile:$log_rotation_{{ name }}
-# end log rotation via outchannel
+# Rotate per hour
+$template RotateHourly_{{ name }},"{{ app_log_dir }}/%$YEAR%/%$MONTH%/%$DAY%/%$HOUR%/{{ name }}.log"
+if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then { action (type="omfile" DynaFile="RotateHourly_{{ name }}" template="RunnableJSON") }
 
-# {{ name }} access file:
-#Add a tag for {{ name }} events
+# Loggly: Add a tag for {{ name }} events
 $template LogglyFormat_{{ name }},"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [{{ loggly_token }}@41058 tag=\"runnable\" tag=\"{{ node_env }}\"] %msg%\n"
-if $msg contains '{{ name }}' then @@logs-01.loggly.com:6514;LogglyFormat_{{ name }}
-if $msg contains '{{ name }}' then stop
+if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then @@logs-01.loggly.com:6514;LogglyFormat_{{ name }}
+if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then stop