logstash json фильтр не разбор. получение _jsonparsefailure хотя JSON правильно

Question

Я пытаюсь разобрать этот JSon сообщение:

{"messageType": "cagw","uniqueID": "8e760b6b7c4e937ebb7063746dd945ecee782cac","sessionID": "6c43b719e0e457784a0a6d5dfef96a82"} 

Хотя формат JSON является правильным, elasticsearch тег как _jsonparsefailure

мой logstash конф:

input { 
syslog { 
    port => 5001 
    type => "syslog" 
    codec => json 
    } 
} 
filter { 
    if [type] == "syslog" { 
    json { 
     source => "message" 
    } 
    } 
} 
output { 
    elasticsearch { 
    } 
} 

Мой Logstash отладочный вывод:

{:timestamp=>"2015-12-17T10:17:51.613000+0000", :message=>"new connection", :client=>"127.0.0.1:54660", :level=>:info, :file=>"logstash/inputs/syslog.rb", :line=>"170", :method=>"tcp_receiver"} 
{:timestamp=>"2015-12-17T10:17:51.625000+0000", :message=>"JSON parse failure. Falling back to plain-text", :error=>#<LogStash::Json::ParserError: Unexpected character ('<' (code 60)): expected a valid value (number, String, array, object, 'true', 'false' or 'null') 
at [Source: [B@41caf092; line: 1, column: 2]>, :data=>"<13>Dec 17 10:17:51 ip-172-31-27-253 ubuntu: {messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", :level=>:info, :file=>"logstash/codecs/json.rb", :line=>"53", :method=>"decode"} 
{:timestamp=>"2015-12-17T10:17:51.629000+0000", :message=>"Running grok filter", :event=>#<LogStash::Event:0x43943d03 @metadata_accessors=#<LogStash::Util::Accessors:0x43e35cfd @store={}, @lut={}>, @cancelled=false, @data={"message"=>"<13>Dec 17 10:17:51 ip-172-31-27-253 ubuntu: {messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1"}, @metadata={}, @accessors=#<LogStash::Util::Accessors:0x6465de36 @store={"message"=>"<13>Dec 17 10:17:51 ip-172-31-27-253 ubuntu: {messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1"}, @lut={"type"=>[{"message"=>"<13>Dec 17 10:17:51 ip-172-31-27-253 ubuntu: {messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1"}, "type"], "host"=>[{"message"=>"<13>Dec 17 10:17:51 ip-172-31-27-253 ubuntu: {messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1"}, "host"]}>>, :level=>:debug, :file=>"logstash/filters/grok.rb", :line=>"283", :method=>"filter"} 
{:timestamp=>"2015-12-17T10:17:51.632000+0000", :message=>"Regexp match object", :names=>["POSINT:priority", "SYSLOGTIMESTAMP:timestamp", "TIMESTAMP_ISO8601:timestamp8601", "NONNEGINT:facility", "NONNEGINT:priority", "SYSLOGHOST:logsource", "PROG:program", "POSINT:pid", "GREEDYDATA:message"], :captures=>["13", "Dec 17 10:17:51", nil, nil, nil, "ip-172-31-27-253", "ubuntu", nil, "{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n"], :level=>:debug, :file=>"grok-pure.rb", :line=>"179", :method=>"match_and_capture"} 
{:timestamp=>"2015-12-17T10:17:51.635000+0000", :message=>"Event now: ", :event=>#<LogStash::Event:0x43943d03 @metadata_accessors=#<LogStash::Util::Accessors:0x43e35cfd @store={}, @lut={}>, @cancelled=false, @data={"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, @metadata={}, @accessors=#<LogStash::Util::Accessors:0x6465de36 @store={"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, @lut={"type"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, "type"], "host"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, "host"], "message"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, "message"], "priority"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, "priority"], "timestamp"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, "timestamp"], "logsource"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, "logsource"], "program"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.627Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>"13", "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu"}, "program"]}>>, :level=>:debug, :file=>"logstash/filters/grok.rb", :line=>"303", :method=>"filter"} 
{:timestamp=>"2015-12-17T10:17:51.636000+0000", :message=>"Date filter: received event", :type=>"syslog", :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"206", :method=>"filter"} 
{:timestamp=>"2015-12-17T10:17:51.636000+0000", :message=>"Date filter looking for field", :type=>"syslog", :field=>"timestamp", :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"209", :method=>"filter"} 
{:timestamp=>"2015-12-17T10:17:51.000Z", :message=>"Date parsing done", :value=>"Dec 17 10:17:51", :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"239", :method=>"filter"} 
{:timestamp=>"2015-12-17T10:17:51.654000+0000", :message=>"filter received", :event=>{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, :level=>:debug, :file=>"(eval)", :line=>"40", :method=>"filter_func"} 
{:timestamp=>"2015-12-17T10:17:51.655000+0000", :message=>"Running json filter", :event=>#<LogStash::Event:0x43943d03 @metadata_accessors=#<LogStash::Util::Accessors:0x43e35cfd @store={}, @lut={}>, @cancelled=false, @data={"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, @metadata={}, @accessors=#<LogStash::Util::Accessors:0x6465de36 @store={"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, @lut={"type"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "type"], "host"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "host"], "message"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "message"], "priority"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "priority"], "timestamp"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "timestamp"], "logsource"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "logsource"], "program"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "program"], "tags"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "tags"], "severity"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "severity"], "facility"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "facility"], "timestamp8601"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "timestamp8601"], "@timestamp"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "@timestamp"], "facility_label"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "facility_label"], "severity_label"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "severity_label"], "[type]"=>[{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, "type"]}>>, :level=>:debug, :file=>"logstash/filters/json.rb", :line=>"58", :method=>"filter"} 
{:timestamp=>"2015-12-17T10:17:51.657000+0000", :message=>"Trouble parsing json", :source=>"message", :raw=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", :exception=>#<LogStash::Json::ParserError: Unexpected character ('m' (code 109)): was expecting double-quote to start field name 
at [Source: [B@6e3bc278; line: 1, column: 3]>, :level=>:warn, :file=>"logstash/filters/json.rb", :line=>"90", :method=>"filter"} 
{:timestamp=>"2015-12-17T10:17:51.658000+0000", :message=>"output received", :event=>{"message"=>"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\n", "tags"=>["_jsonparsefailure"], "@version"=>"1", "@timestamp"=>"2015-12-17T10:17:51.000Z", "type"=>"syslog", "host"=>"127.0.0.1", "priority"=>13, "timestamp"=>"Dec 17 10:17:51", "logsource"=>"ip-172-31-27-253", "program"=>"ubuntu", "severity"=>5, "facility"=>1, "facility_label"=>"user-level", "severity_label"=>"Notice"}, :level=>:debug, :file=>"(eval)", :line=>"46", :method=>"output_func"} 
{:timestamp=>"2015-12-17T10:17:51.731000+0000", :message=>"Flushing buffer at interval", :instance=>"#<LogStash::Outputs::ElasticSearch::Buffer:0x27906bc0 @operations_mutex=#<Mutex:0x29579142>, @max_size=500, @operations_lock=#<Java::JavaUtilConcurrentLocks::ReentrantLock:0x2d6800b0>, @submit_proc=#<Proc:0x4c0023df@/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-2.2.0-java/lib/logstash/outputs/elasticsearch/common.rb:55>, @logger=#<Cabin::Channel:0x59c8769c @metrics=#<Cabin::Metrics:0x34489095 @metrics_lock=#<Mutex:0x8456436>, @metrics={}, @channel=#<Cabin::Channel:0x59c8769c ...>>, @subscriber_lock=#<Mutex:0x4516a03a>, @level=:debug, @subscribers={12590=>#<Cabin::Outputs::IO:0x2f4cbdd1 @io=#<IO:fd 1>, @lock=#<Mutex:0xc0b18da>>}, @data={}>, @last_flush=2015-12-17 10:17:50 +0000, @flush_interval=1, @stopping=#<Concurrent::AtomicBoolean:0x7cec6e11>, @buffer=[[\"index\", {:_id=>nil, :_index=>\"test\", :_type=>\"syslog\", :_routing=>nil}, #<LogStash::Event:0x43943d03 @metadata_accessors=#<LogStash::Util::Accessors:0x43e35cfd @store={}, @lut={}>, @cancelled=false, @data={\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, @metadata={}, @accessors=#<LogStash::Util::Accessors:0x6465de36 @store={\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, @lut={\"type\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"type\"], \"host\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"host\"], \"message\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"message\"], \"priority\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"priority\"], \"timestamp\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"timestamp\"], \"logsource\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"logsource\"], \"program\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"program\"], \"tags\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"tags\"], \"severity\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"severity\"], \"facility\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"facility\"], \"timestamp8601\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"timestamp8601\"], \"@timestamp\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"@timestamp\"], \"facility_label\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"facility_label\"], \"severity_label\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"severity_label\"], \"[type]\"=>[{\"message\"=>\"{messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}\\n\", \"tags\"=>[\"_jsonparsefailure\"], \"@version\"=>\"1\", \"@timestamp\"=>\"2015-12-17T10:17:51.000Z\", \"type\"=>\"syslog\", \"host\"=>\"127.0.0.1\", \"priority\"=>13, \"timestamp\"=>\"Dec 17 10:17:51\", \"logsource\"=>\"ip-172-31-27-253\", \"program\"=>\"ubuntu\", \"severity\"=>5, \"facility\"=>1, \"facility_label\"=>\"user-level\", \"severity_label\"=>\"Notice\"}, \"type\"]}>>]], @flush_thread=#<Thread:0x2155d7cc run>>", :interval=>1, :level=>:info, :file=>"logstash/outputs/elasticsearch/buffer.rb", :line=>"90", :method=>"interval_flush"} 

Спасибо, Юваль

Answer 1

Поскольку у вас есть jsoncodec во входной фильтр syslog, вам не нужно, чтобы добавить дополнительный json фильтр. Ваше мероприятие уже находится в формате JSON. Что происходит, так это то, что в вашем json фильтрах вы пытаетесь найти поле message, которого не существует.

Вы можете удалить фильтр json или удалить настройки codec в свой входной файл syslog и он будет работать.

UPDATE

сообщение об ошибке вы получаете LogStash::Json::ParserError: Unexpected character ('<' (code 60)) что означает, что ваш вывод системного журнала не является допустимым JSON, так что вы не можете использовать параметр codec => json, и вы, безусловно, нужны json фильтра, но для этого нужно разобрать правильное поле (т.е. не message).

Ваши бревенчатые линии выглядят как

<13>Dec 17 10:17:51 ip-172-31-27-253 ubuntu: {messageType: cagw,uniqueID: 8e760b6b7c4e937ebb7063746dd945ecee782cac,sessionID: 6c43b719e0e457784a0a6d5dfef96a82}