nginx -> rsyslog -> rsyslog-elasticsearch -> elasticsearch¶

apt install rsyslog-elasticsearch

module(load="omelasticsearch")
module(load="mmjsonparse")

template(name="template_nginx_yyyy_mm_dd" type="list") {
    constant(value="nginx-")
    property(name="timereported" dateformat="year")
    constant(value=".")
    property(name="timereported" dateformat="month")
    constant(value=".")
    property(name="timereported" dateformat="day")
}

template(name="template_msg_to_json" type="list") {
    property(name="$!msg")
}

if ($programname == "nginx") then {
    action(type="mmjsonparse")
    action(type="omelasticsearch" 
        server=["https://es1.domain.tld:443", "https://es2.domain.tld:443", "https://es3.domain.tld:443"]
        uid="elastic" 
        pwd="secret" 
        template="template_msg_to_json" 
        dynSearchIndex="on" 
        searchIndex="template_nginx_yyyy_mm_dd" 
        searchType="nginx" 
        bulkmode="on" 
        maxbytes="100m" 
        queue.type="linkedlist" 
        queue.size="5000" 
        queue.dequeuebatchsize="300" 
        action.resumeretrycount="-1" 
    )
}

nginx/conf.d/log_format.conf¶

1. access_log syslog:server=unix:/dev/log,tag=nginx,nohostname default_json;

log_format default_json escape=json
    '{'
        '"args":"$args",'
        '"bytes_sent":"$bytes_sent",'
        '"connection": "$connection",'
        '"connection_requests": "$connection_requests",'
        '"hostname": "$hostname",'
        '"http_referrer":"$http_referer",'
        '"http_user_agent":"$http_user_agent",'
        '"remote_addr":"$remote_addr",'
        '"remote_port":"$remote_port",'
        '"remote_user":"$remote_user",'
        '"request":"$request",'
        '"request_length":"$request_length",'
        '"request_method":"$request_method",'
        '"request_time":"$request_time",'
        '"scheme": "$scheme",'
        '"server_name": "$server_name",'
        '"server_protocol": "$server_protocol",'
        '"status": "$status",'
        '"time_local":"$time_iso8601",'
        '"upstream_addr": "$upstream_addr",'
        '"upstream_response_time": "$upstream_response_time",'
        '"upstream_status": "$upstream_status",'
        '"uri": "$uri"'
    '}';

rsyslog -> elasticsearch¶

# elasticsearch
module(load="omelasticsearch")
template(name="elastic_date_template" type="list") {
    constant(value="rsyslog-")
    property(name="timereported" dateformat="year")
    constant(value=".")
    property(name="timereported" dateformat="month")
    constant(value=".")
    property(name="timereported" dateformat="day")
}

template(name="elastic_msg_template" type="list" option.json="on") {
    constant(value="{")
    constant(value="\"timestamp\":\"")      property(name="timereported" dateFormat="rfc3339")
    constant(value="\",\"message\":\"")     property(name="msg")
    constant(value="\",\"host\":\"")        property(name="hostname")
    constant(value="\",\"severity\":\"")    property(name="syslogseverity-text")
    constant(value="\",\"facility\":\"")    property(name="syslogfacility-text")
    constant(value="\",\"syslogtag\":\"")   property(name="syslogtag")
    constant(value="\",\"programname\":\"") property(name="programname")
    constant(value="\",\"procid\":\"")      property(name="procid")
    constant(value="\"}")
}

action(
    type="omelasticsearch" 
    server="127.0.0.1" 
    serverport="9200" 
    usehttps="off" 
    uid="elastic" 
    pwd="mypass1" 
    template="elastic_msg_template" 
    dynSearchIndex="on" 
    searchIndex="elastic_date_template" 
    searchType="rsyslog" 
    bulkmode="on" 
    maxbytes="100m" 
    queue.type="linkedlist" 
    queue.size="5000" 
    queue.dequeuebatchsize="300" 
    action.resumeretrycount="-1" 
)