База знаний

Wiki »

Logstash » История » Редакция 6

Редакция 5 (Константин Пильник, 2022-11-29 18:33) → Редакция 6/7 (Константин Пильник, 2022-11-29 18:35)

h1. Logstash 

 h2. разбираем лог syslog 

 h4. rsyslog 

 <pre><code class="bash"> 
 ~# cat /etc/rsyslog.d/remote.conf  
 # udp 
 *.* @127.0.0.1:500 
 # tcp 
 #*.* @@127.0.0.1:500 
 </code></pre> 

 {{collapse(logstash.conf) 
 <pre><code class="php"> 
 input { 
   file { 
     type => "message" 
     path => [ "/var/log/messages" ] 
     start_position => "end" 
     stat_interval => 1 
     discover_interval => 3 
   } 
 # production 
 #    udp { 
 #      port => 500 
 #      type => syslog 
 #    } 
 } 

 filter { 
   if [type] == "message" { 
     grok { 
       match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:\s+%{GREEDYDATA:syslog_message}" } 
     } 
     date { 
       match => [ "syslog_timestamp", "MMM    d HH:mm:ss", "MMM dd HH:mm:ss" ] 
     } 
   } 
 } 

 output { 
   stdout {} 
 # production 
 # elasticsearch { 
 #     hosts      => ["elasticsearch:9200"] 
 #     index      => "%{[type]}-%{+YYYY.MM.dd}" 
 #     user       => "${ELASTIC_USERNAME}" 
 #     password => "${ELASTIC_PASSWORD}" 
 # } 
 } 
 </code></pre> 
 }} 

 h2. разбираем лог nginx из файловой системы 

 {{collapse(nginx.conf) 
 <pre> 
 log_format json_combined escape=json 
   '{' 
     '"time_local":"$time_local",' 
     '"remote_addr":"$remote_addr",' 
     '"remote_user":"$remote_user",' 
     '"request":"$request",' 
     '"status": "$status",' 
     '"body_bytes_sent":"$body_bytes_sent",' 
     '"request_time":"$request_time",' 
     '"http_referrer":"$http_referer",' 
     '"http_user_agent":"$http_user_agent"' 
   '}'; 
 access_log /var/log/nginx/access.log json_combined; 
 </pre> 
 }} 
 {{collapse(logstash.conf) 
 <pre><code class="php"> 
 input { 
   file { 
     type => "nginx" 
     path => [ "/var/log/nginx/access.log" ] 
     start_position => "end" 
   } 
 } 

 filter { 
   if [type] == "nginx" { 
     json { 
       source => "message" 
     } 
   } 
 } 

 output { 
   stdout {} 
 } 
 </code></pre> 
 }} 

 h2. разбираем лог nginx из syslog 

 {{collapse(nginx.conf) 
 <pre> 
 log_format json_combined escape=json 
   '{' 
     '"time_local":"$time_local",' 
     '"remote_addr":"$remote_addr",' 
     '"remote_user":"$remote_user",' 
     '"request":"$request",' 
     '"status": "$status",' 
     '"body_bytes_sent":"$body_bytes_sent",' 
     '"request_time":"$request_time",' 
     '"http_referrer":"$http_referer",' 
     '"http_user_agent":"$http_user_agent"' 
   '}'; 
 access_log syslog:server=unix:/dev/log,tag=nginx,severity=error,nohostname json_combined; 
 </pre> 
 }} 
 {{collapse(logstash.conf) 
 <pre><code class="php"> 
 input { 
   file { 
     type => "syslog" 
     path => [ "/var/log/syslog" ] 
     start_position => "end" 
   } 
 } 

 filter { 
   if [type] == "syslog" { 
     grok { 
       match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:\s+%{GREEDYDATA:syslog_message}" } 
     } 
     json { 
       source => "[syslog_message]" 
       target => "[nginx_message]" 
     } 
   } 
 } 

 output { 
   stdout {} 
 } 
 </code></pre> 
 }} 


 h3. ссылки 

 https://www.elastic.co/guide/en/logstash/current/input-plugins.html 
 https://www.elastic.co/guide/en/logstash/current/filter-plugins.html 
 https://www.elastic.co/guide/en/logstash/current/output-plugins.html