Logstash 解析Json字符串，删除json嵌套字段

一、场景：此文以一个简单的json字符串文件为例，描述如何用logstash解析嵌套的json，并删除其中的某些字段

我们在linux中test.json的内容如下：

{"timestamp":"2018-08-02T14:42:50.084467+0800","flow_id":364959073190719,"in_iface":"eth1","event_type":"alert","src_ip":"10.0.0.4","src_port":80,"dest_ip":"10.0.0.5","dest_port":16781,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2101201,"rev":10,"signature":"GPL WEB_SERVER 403 Forbidden","category":"Attempted Information Leak","severity":2},"http":{"hostname":"bapi.yahoo.com","url":"\/v1tns\/searchorderlist?_time=1533192163978","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","xff":"39.106.108.38","http_content_type":"text\/html","http_method":"POST","protocol":"HTTP\/1.0","status":403,"length":568},"app_proto":"http","flow":{"pkts_toserver":5,"pkts_toclient":5,"bytes_toserver":1547,"bytes_toclient":1076,"start":"2018-08-02T14:42:50.082751+0800"}}

为了方便查看，formate后，为如下格式

1{  
2   "timestamp":"2018-08-02T14:42:50.084467+0800",
3   "flow_id":364959073190719,
4   "in_iface":"eth1",
5   "event_type":"alert",
6   "src_ip":"10.0.0.4",
7   "src_port":80,
8   "dest_ip":"10.0.0.5",
9   "dest_port":16781,
10   "proto":"TCP",
11   "tx_id":0,
12   "alert":{  
13      "action":"allowed",
14      "gid":1,
15      "signature_id":2101201,
16      "rev":10,
17      "signature":"GPL WEB_SERVER 403 Forbidden",
18      "category":"Attempted Information Leak",
19      "severity":2
20   },
21   "http":{  
22      "hostname":"bapi.yahoo.com",
23      "url":"\/v1tns\/searchorderlist?_time=1533192163978",
24      "http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36",
25      "xff":"39.106.108.38",
26      "http_content_type":"text\/html",
27      "http_method":"POST",
28      "protocol":"HTTP\/1.0",
29      "status":403,
30      "length":568
31   },
32   "app_proto":"http",
33   "flow":{  
34      "pkts_toserver":5,
35      "pkts_toclient":5,
36      "bytes_toserver":1547,
37      "bytes_toclient":1076,
38      "start":"2018-08-02T14:42:50.082751+0800"
39   }
40}

二、目的： 我们需要解析这个json，并且删除json中**"src_ip"字段和"http下的hostname"**这个字段

我的配置文件如下：

1input {
2    file {
3        path => "/usr/share/logstash/private.cond/nestjson.json"
4        codec => "json"
5        start_position => "beginning"
6        sincedb_path => "/dev/null"
7    }
8}
9filter {
10    json {
11       source => "message"
12    }
13    mutate {
14      remove_field => ["src_ip","[http][hostname]"]
15   }
16}
17output {
18    stdout {
19      codec => rubydebug
20    }
21}
22注意第14行删除

注意第14行删除字段和嵌套字段的写法

运行logstash我们得到如下输出：

1{
2         "alert" => {
3                 "gid" => 1,
4                 "rev" => 10,
5            "severity" => 2,
6           "signature" => "GPL WEB_SERVER 403 Forbidden",
7              "action" => "allowed",
8        "signature_id" => 2101201,
9            "category" => "Attempted Information Leak"
10    },
11          "http" => {
12                 "protocol" => "HTTP/1.0",
13        "http_content_type" => "text/html",
14          "http_user_agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
15              "http_method" => "POST",
16                   "length" => 568,
17                      "url" => "/v1tns/searchorderlist?_time=1533192163978",
18                      "xff" => "39.106.108.38",
19                   "status" => 403
20    },
21          "path" => "/usr/share/logstash/private.cond/test.json",
22    "event_type" => "alert",
23      "src_port" => 80,
24     "dest_port" => 16781,
25       "dest_ip" => "10.0.0.5",
26         "proto" => "TCP",
27       "flow_id" => 364959073190719,
28         "tx_id" => 0,
29      "@version" => "1",
30      "in_iface" => "eth1",
31     "timestamp" => "2018-08-02T14:42:50.084467+0800",
32          "flow" => {
33         "pkts_toserver" => 5,
34         "pkts_toclient" => 5,
35        "bytes_toserver" => 1547,
36                 "start" => "2018-08-02T14:42:50.082751+0800",
37        "bytes_toclient" => 1076
38    },
39          "host" => "elk",
40     "app_proto" => "http",
41    "@timestamp" => 2018-08-02T10:14:14.372Z
42}

我们可以看到src_ip和http下的hostname已经被成功删除