Logstash收集nginx访问日志和错误日志

1、收集访问日志

1）、首先是要在nginx里面配置日志格式化输出

1log_format  main  "$http_x_forwarded_for | $time_local | $request | $status | $body_bytes_sent | $request_body | $content_length | $http_referer | $http_user_agent |"
2                      "$http_cookie | $remote_addr | $hostname | $upstream_addr | $upstream_response_time | $request_time" ;
3
4    access_log  /var/log/nginx/access.log  main;

2）、接下来开始在logstash创建处理nginx的配置文件

1input {
2        file {
3                path => ["/var/log/nginx/access.log"]
4        }
5}
6
7filter {
8        ruby {
9                init => "@kname =['http_x_forwarded_for','time_local','request','status','body_bytes_sent','request_body','content_length','http_referer','http_user_agent','http_cookie','remote_addr','hostname','upstream_addr','upstream_response_time','request_time']"
10                code => "new_event = LogStash::Event.new(Hash[@kname.zip(event.get('message').split('|'))])
11                new_event.remove('@timestamp')
12                event.append(new_event)
13                "
14        }
15
16if [request] {
17        ruby {
18                init => "@kname = ['method','uri','verb']"
19                code => "
20                        new_event = LogStash::Event.new(Hash[@kname.zip(event.get('request').split(' '))])
21                        new_event.remove('@timestamp')
22                        event.append(new_event)
23                "
24        }
25 } 
26if [uri] {
27        ruby{
28                init => "@kname = ['url_path','url_args']"
29                code => "
30                        new_event = LogStash::Event.new(Hash[@kname.zip(event.get('uri').split('?'))])
31                        new_event.remove('@timestamp')
32                        event.append(new_event)
33                "
34        }
35 }
36kv {
37        prefix =>"url_"
38        source =>"url_args"
39        field_split =>"&"
40        include_keys => ["uid","cip"]
41        remove_field => ["url_args","uri","request"]
42}
43mutate {
44        convert => [
45                "body_bytes_sent","integer",
46                "content_length","integer",
47                "upstream_response_time","float",
48                "request_time","float"
49        ]
50 }
51date {
52        match => [ "time_local","dd/MMM/yyyy:hh:mm:ss Z" ]
53        locale => "en"
54 }
55}
56output{stdout{}}

此处的例子借鉴ELKstack权威指南里面的例子，不过书中的例子有错，我这里修改好了，可以参考书籍39页和66页

github:https://github.com/weixinqing/Logstash-example/blob/master/initnginx.conf

3）、最后允许一下看一下效果所示：

1{
2                  "url_path" => "/",
3           "body_bytes_sent" => 0,
4                  "@version" => "1",
5                   "message" => "- | 05/Mar/2019:16:21:40 +0800 | GET / HTTP/1.1 | 304 | 0 | - | - | - | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0 |- | 172.16.0.10 | elk-chaofeng07 | - | - | 0.000",
6                      "host" => "ELK-chaofeng07",
7               "http_cookie" => "- ",
8             "upstream_addr" => " - ",
9    "upstream_response_time" => 0.0,
10                "@timestamp" => 2019-03-05T08:21:41.352Z,
11                       "uri" => "/",
12                   "request" => " GET / HTTP/1.1 ",
13                      "path" => "/var/log/nginx/access.log",
14                  "url_args" => nil,
15                  "hostname" => " elk-chaofeng07 ",
16                      "verb" => "HTTP/1.1",
17           "http_user_agent" => " Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0 ",
18                "time_local" => " 05/Mar/2019:16:21:40 +0800 ",
19              "request_body" => " - ",
20               "remote_addr" => " 172.16.0.10 ",
21                    "status" => " 304 ",
22              "request_time" => 0.0,
23                    "method" => "GET",
24              "http_referer" => " - ",
25                      "tags" => [
26        [0] "_dateparsefailure"
27    ],
28            "content_length" => 0,
29      "http_x_forwarded_for" => "- "
30}

唯一不足的就是中间报了个错误，可以自行解决一下。

2、收集错误日志

定义logstash处理的配置文件

1input{
2        file {
3                path => ["/var/log/nginx/error.log"]
4        }
5}
6filter{
7        grok {
8                match => {"message" => "(?<datetime>\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d) \[(?<errortype>\w+)\] \S+: \*\d+ (?<errormsg>[^,]+), \w+: %{IP:remotehost}, \w+: \w+, \w+: (?<request>[^,]+), \w+: \"%{IP:localhost}\""}
9        }
10        mutate {
11                remove_field => ["message"]
12        }
13if [request] {
14        ruby {
15                init => "@kname = ['method','uri','verb']"
16                code => "
17                        new_event = LogStash::Event.new(Hash[@kname.zip(event.get('request').split(' '))])
18                        new_event.remove('@timestamp')
19                        event.append(new_event)
20                "
21        }
22}
23
24}
25output{stdout{}}

查看一下效果：

1{
2      "@version" => "1",
3          "path" => "/var/log/nginx/error.log",
4    "remotehost" => "172.16.0.10",
5       "request" => "\"GET /8 HTTP/1.1\"",
6          "verb" => "HTTP/1.1\"",
7           "uri" => "/8",
8          "host" => "ELK-chaofeng07",
9     "localhost" => "172.16.0.57",
10        "method" => "\"GET",
11    "@timestamp" => 2019-03-05T10:43:54.377Z,
12      "datetime" => "2019/03/05 18:43:53",
13      "errormsg" => "open() \"/usr/share/nginx/html/8\" failed (2: No such file or directory)",
14     "errortype" => "error"
15}