Haproxy logstash filter fix for headers and parsefailures
@@ -40,11 +40,15 @@ filter {
if " [ TCP] " in [message] {
drop { }
}
+ if " [WARNING] " in [message] {
+ drop { }
+ }
+
# Sometimes HAproxy will report an SSL handshake failure, using a different log line. We check for that as well
if "SSL handshake failure" in [message] {
grok {
- match => { "message"=> [ "%{SYSLOGTIMESTAMP} %{IPORHOST:redir.host} %{SYSLOGPROG}: %{IPORHOST:source.ip}:%{POSINT:source.port} \[(?<redir.traffic.timestamp>%{MONTHDAY}\/%{MONTH}\/%{YEAR}:%{TIME})] (?<redir.frontend>([^/]*))%{GREEDYDATA:redir.traffic.request}" ] }
+ match => { "message"=> [ "%{SYSLOGTIMESTAMP} %{IPORHOST:redir.host} %{SYSLOGPROG}: %{IPORHOST:source.ip}:%{POSINT:source.port} \[(?<redir.traffic.timestamp>%{MONTHDAY}\/%{MONTH}\/%{YEAR}:%{TIME})\] (?<redir.frontend>([^/]*))%{GREEDYDATA:redir.traffic.request}" ] }
}
mutate {
add_field => { "redir.destination" => "ssl-error" }
@@ -71,9 +75,6 @@ filter {
grok {
match => { "message" => [ "%{SYSLOGTIMESTAMP} %{IPORHOST:redir.host} %{SYSLOGPROG}:(%{GREEDYDATA})? frontend:(?<redir.frontend>([^/]*))/(([^/]*))/%{IPORHOST:redir.frontendip}:%{POSINT:redir.frontendport} backend:%{NOTSPACE:redir.destination} client:%{IPORHOST:source.ip}:%{POSINT:source.port} GMT:%{HTTPDATE:redir.traffic.timestamp} useragent:(?<redir.traffic.useragent>(.*)) xforwardedfor:- headers:\{\|(?<redir.traffic.headers>([^\}]*))} request:%{GREEDYDATA:redir.traffic.request}" ] }
}
- mutate {
- split => { "redir.traffic.headers" => "|" }
- }
} else if "request:" in [message] {
# Lines with X-Forwarded-For set. We already filtered out the 'xfordwardedfor:-', so anything left with a large enough log line should be good
grok {
@@ -87,6 +88,13 @@ filter {
}
}
+ # Split the values in the headers filed
+ if [redir.traffic.headers] {
+ mutate {
+ split => { "redir.traffic.headers" => "|" }
+ }
+ }
+
# Set the timestamp from the log to @timestamp, example: 15/Apr/2018:19:22:31 +0000
date {
match => [ "redir.traffic.timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
@@ -116,6 +124,13 @@ filter {
]
}
}
+ if "ffff" in [redir.frontendip] {
+ mutate {
+ gsub => [
+ "redir.frontendip", "\:\:ffff\:", ""
+ ]
+ }
+ }
# Add data to the source.ip
if [source.ip] {
