How to config for sending Swift SAIO logs to ELK5 via Rsyslog

This post is sharing my task for sending logs from rsyslog client ( Swift SAIO in this case ) to rsyslog server ( elk5 : ElasticStack 5.0 ).

1. Enable Receiver ( rsyslog-server ) setup at elk5 container

a. Update elk5 ( receiver ) rsyslog.conf# sudo vi /etc/rsyslog.conf

    Change
from
# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514
# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514

To
# provides UDP syslog reception
$ModLoad imudp
$UDPServerRun 514
# provides TCP syslog reception
$ModLoad imtcp
$InputTCPServerRun 514

b. Restart rsyslog service

# sudo systemctl restart rsyslog

2. Enable sending-end ( rsyslog-client ) setup at Swift node.

a. Add elk5 server ip in rsyslog-client ( swift node )

    # cd /etc/rsyslog.d/
You should see there has 0-swift.conf already

# sudo vi 0-swift.conf and add this line @elk5_container_ip:514

PS: @ is UDP , @@ is TCP

# $ cat 0-swift.conf
# NOTE: we used to enable UDP logging here, but we switched
# back to just unix domain socket.

$imjournalRatelimitInterval 60
$imjournalRatelimitBurst 600000

*.*                         @192.168.81.104:514

# Log all Swift proxy-server access log lines (local2) to
# /var/log/swift/proxy_access.log
local2.* /var/log/swift/proxy_access.log;RSYSLOG_FileFormat

# Log all Swift lines to /var/log/swift/all.log
# AND PREVENT FURTHER LOGGING OF THEM (eg. to /var/log/syslog)
local0.*;local2.* /var/log/swift/all.log;RSYSLOG_TraditionalFileFormat
& ~

  
b. Restart the rsyslog client service

# sudo systemctl restart rsyslog.serviceOr # sudo service rsyslog restart

3. Formatting the Log Data to JSON at elk5 container.

Elasticsearch requires that all documents it receives be in JSON format.

a. Add json template 

# sudo vi /etc/rsyslog.d/01-json-template.confOr# cat /etc/rsyslog.d/01-json-template.conftemplate(name="json-template"  type="list") {

constant(value="{")      constant(value="\"@timestamp\":\"")     property(name="timereported" dateFormat="rfc3339")      constant(value="\",\"@version\":\"1")      constant(value="\",\"message\":\"")     property(name="msg" format="json")      constant(value="\",\"sysloghost\":\"")  property(name="hostname")      constant(value="\",\"severity\":\"")    property(name="syslogseverity-text")      constant(value="\",\"facility\":\"")    property(name="syslogfacility-text")      constant(value="\",\"programname\":\"") property(name="programname")      constant(value="\",\"procid\":\"")      property(name="procid")    constant(value="\"}\n")

}

4. Configuring the Receiver ( Rsyslog-Server ) rsyslog output for logstash at elk5 container.

a. Configure template

    # sudo vi /etc/rsyslog.d/60-output.conf
Or
# cat /etc/rsyslog.d/60-output.conf

# This line sends all lines to defined IP address at port 10514,
# using the "json-template" format template

*.*                         @localhost:10514;json-template

  

5. Configure Logstash to Receive JSON output at elk5 container

a. Isntall the security key for the logstash repository

# wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -

b. Add repository definition to your /etc/apt/sources.list

# echo "deb http://packages.elastic.co/logstash/2.3/debian stable main" | sudo tee -a /etc/apt/sources.list

c. Do apt update

# apt-get update

PS: I'm not sure 5.a, 5.b and 5.c doesn't matter, I suspect whether we need it or not.

d. Add new logstash.conf

    # vi /etc/logstash/conf.d/logstash.conf

    Or
# cat /etc/logstash/conf.d/logstash.conf

# This input block will listen on port 10514 for logs to come in.
# host should be an IP on the Logstash server.
# codec => "json" indicates that we expect the lines we're receiving to be in JSON format
# type => "rsyslog" is an optional identifier to help identify messaging streams in the pipeline.

input {
  udp {
    host => "localhost"
    port => 10514
    codec => "json"
    type => "rsyslog"
  }
}

# This is an empty filter block.  You can later add other filters here to further process
# your log lines

filter { }

# This output block will send all events of type "rsyslog" to elasticsearch at the configured
# host and port into daily indices of the pattern, "rsyslog-YYYY.MM.DD"

output {
  if [type] == "rsyslog" {
    elasticsearch {
      hosts => [ "localhost:9200" ]
    }
  }
}

e. Move previous 02/10/30*.conf to /tmp or some backup folder, because we don't need it.

    #root@elk5-u1604:/tmp/logstash_conf_backup# ll
total 6
drwxr-xr-x 2 root root   5 Nov  4 23:36 ./
drwxrwxrwx 6 root root   6 Nov  5 00:17 ../
-rw-r--r-- 1 root root  41 Nov  4 21:35 02-beats-input.conf
-rw-r--r-- 1 root root 456 Nov  4 21:35 10-syslog-filter.conf
-rw-r--r-- 1 root root 210 Nov  4 21:35 30-elasticsearch-output.conf

f. Restart logstash

# sudo systemctl restart logstashor# sudo service logstash restart

g. Restart rsyslog

# sudo systemctl restart rsyslogor# sudo service rsyslog restart

h. Double check the ports

    root@elk5-u1604:/tmp/logstash_conf_backup# netstat -na | grep 10514

    udp6       0      0 127.0.0.1:10514         :::*
root@elk5-u1604:/tmp/logstash_conf_backup# netstat -ntlp | grep LISTEN
(Not all processes could be identified, non-owned process info
 will not be shown, you would have to be root to see it all.)
tcp        0      0 127.0.0.1:5601          0.0.0.0:*               LISTEN      5601/node
tcp        0      0 0.0.0.0:514             0.0.0.0:*               LISTEN      -
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      5429/nginx -g daemo
tcp6       0      0 127.0.0.1:9600          :::*                    LISTEN      8628/java
tcp6       0      0 :::514                  :::*                    LISTEN      -
tcp6       0      0 127.0.0.1:9200          :::*                    LISTEN      8459/java
tcp6       0      0 ::1:9200                :::*                    LISTEN      8459/java
tcp6       0      0 127.0.0.1:9300          :::*                    LISTEN      8459/java
tcp6       0      0 ::1:9300                :::*                    LISTEN      8459/java