База знаний

Wiki »

Logstash » История » Редакция 2

Редакция 1 (Константин Пильник, 2022-10-14 13:48) → Редакция 2/7 (Константин Пильник, 2022-10-14 13:48)

h1. Logstash 

 h2. разбираем лог syslog 

 {{collapse(logstash.conf) 
 input { 
   file { 
     type => "message" 
     path => [ "/var/log/messages" ] 
     start_position => "end" 
     stat_interval => 1 
     discover_interval => 3 
   } 
 } 

 filter { 
   if [type] == "message" { 
     grok { 
       match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:\s+%{GREEDYDATA:syslog_message}" } 
     } 
     date { 
       match => [ "syslog_timestamp", "MMM    d HH:mm:ss", "MMM dd HH:mm:ss" ] 
     } 
   } 
 } 

 output { 
   stdout {} 
 } 
 }} 

 h2. разбираем лог nginx из файловой системы 

 {{collapse(nginx.conf) 
 <pre> 
 log_format json_combined escape=json 
   '{' 
     '"time_local":"$time_local",' 
     '"remote_addr":"$remote_addr",' 
     '"remote_user":"$remote_user",' 
     '"request":"$request",' 
     '"status": "$status",' 
     '"body_bytes_sent":"$body_bytes_sent",' 
     '"request_time":"$request_time",' 
     '"http_referrer":"$http_referer",' 
     '"http_user_agent":"$http_user_agent"' 
   '}'; 
 access_log /var/log/nginx/access.log json_combined; 
 </pre> 
 }} 
 {{collapse(logstash.conf) 
 <pre><code class="javascript"> 
 input { 
   file { 
     type => "nginx" 
     path => [ "/var/log/nginx/access.log" ] 
     start_position => "end" 
   } 
 } 

 filter { 
   if [type] == "nginx" { 
     json { 
       source => "message" 
     } 
   } 
 } 

 output { 
   stdout {} 
 } 
 </code></pre> 
 }} 

 h2. разбираем лог nginx из syslog 

 {{collapse(nginx.conf) 
 <pre> 
 log_format json_combined escape=json 
   '{' 
     '"time_local":"$time_local",' 
     '"remote_addr":"$remote_addr",' 
     '"remote_user":"$remote_user",' 
     '"request":"$request",' 
     '"status": "$status",' 
     '"body_bytes_sent":"$body_bytes_sent",' 
     '"request_time":"$request_time",' 
     '"http_referrer":"$http_referer",' 
     '"http_user_agent":"$http_user_agent"' 
   '}'; 
 access_log syslog:server=unix:/dev/log,tag=nginx,severity=error,nohostname json_combined; 
 </pre> 
 }} 
 {{collapse(logstash.conf) 
 input { 
   file { 
     type => "syslog" 
     path => [ "/var/log/syslog" ] 
     start_position => "end" 
   } 
 } 

 filter { 
   if [type] == "syslog" { 
     grok { 
       match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:\s+%{GREEDYDATA:syslog_message}" } 
     } 
     json { 
       source => "[syslog_message]" 
       target => "[nginx_message]" 
     } 
   } 
 } 

 output { 
   stdout {} 
 } 
 }} 


 h3. ссылки 

 https://www.elastic.co/guide/en/logstash/current/input-plugins.html 
 https://www.elastic.co/guide/en/logstash/current/filter-plugins.html 
 https://www.elastic.co/guide/en/logstash/current/output-plugins.html