Проект

Общее

Профиль

Logstash » История » Версия 3

Константин Пильник, 2022-10-14 13:50

1 1 Константин Пильник
h1. Logstash
2
3
h2. разбираем лог syslog
4
5
{{collapse(logstash.conf)
6 3 Константин Пильник
<pre><code class="javascript">
7 1 Константин Пильник
input {
8
  file {
9
    type => "message"
10
    path => [ "/var/log/messages" ]
11
    start_position => "end"
12
    stat_interval => 1
13
    discover_interval => 3
14
  }
15
}
16
17
filter {
18
  if [type] == "message" {
19
    grok {
20
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:\s+%{GREEDYDATA:syslog_message}" }
21
    }
22
    date {
23
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
24
    }
25
  }
26
}
27
28
output {
29
  stdout {}
30
}
31 3 Константин Пильник
</code></pre>
32 1 Константин Пильник
}}
33
34
h2. разбираем лог nginx из файловой системы
35
36
{{collapse(nginx.conf)
37
<pre>
38
log_format json_combined escape=json
39
  '{'
40
    '"time_local":"$time_local",'
41
    '"remote_addr":"$remote_addr",'
42
    '"remote_user":"$remote_user",'
43
    '"request":"$request",'
44
    '"status": "$status",'
45
    '"body_bytes_sent":"$body_bytes_sent",'
46
    '"request_time":"$request_time",'
47
    '"http_referrer":"$http_referer",'
48
    '"http_user_agent":"$http_user_agent"'
49
  '}';
50
access_log /var/log/nginx/access.log json_combined;
51
</pre>
52
}}
53
{{collapse(logstash.conf)
54
<pre><code class="javascript">
55
input {
56
  file {
57
    type => "nginx"
58
    path => [ "/var/log/nginx/access.log" ]
59
    start_position => "end"
60
  }
61
}
62
63
filter {
64
  if [type] == "nginx" {
65
    json {
66
      source => "message"
67
    }
68
  }
69
}
70
71
output {
72
  stdout {}
73
}
74
</code></pre>
75
}}
76
77
h2. разбираем лог nginx из syslog
78
79
{{collapse(nginx.conf)
80
<pre>
81
log_format json_combined escape=json
82
  '{'
83
    '"time_local":"$time_local",'
84
    '"remote_addr":"$remote_addr",'
85
    '"remote_user":"$remote_user",'
86
    '"request":"$request",'
87
    '"status": "$status",'
88
    '"body_bytes_sent":"$body_bytes_sent",'
89
    '"request_time":"$request_time",'
90
    '"http_referrer":"$http_referer",'
91
    '"http_user_agent":"$http_user_agent"'
92
  '}';
93
access_log syslog:server=unix:/dev/log,tag=nginx,severity=error,nohostname json_combined;
94
</pre>
95
}}
96
{{collapse(logstash.conf)
97 3 Константин Пильник
<pre><code class="javascript">
98 1 Константин Пильник
input {
99
  file {
100
    type => "syslog"
101
    path => [ "/var/log/syslog" ]
102
    start_position => "end"
103
  }
104
}
105
106
filter {
107
  if [type] == "syslog" {
108
    grok {
109
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:\s+%{GREEDYDATA:syslog_message}" }
110
    }
111
    json {
112
      source => "[syslog_message]"
113
      target => "[nginx_message]"
114
    }
115
  }
116
}
117
118
output {
119
  stdout {}
120
}
121 3 Константин Пильник
</code></pre>
122 1 Константин Пильник
}}
123
124
125 2 Константин Пильник
h3. ссылки
126
127 1 Константин Пильник
https://www.elastic.co/guide/en/logstash/current/input-plugins.html
128
https://www.elastic.co/guide/en/logstash/current/filter-plugins.html
129
https://www.elastic.co/guide/en/logstash/current/output-plugins.html