Elkstack Real-time analysis of Haproxy access log configuration

1.Haproxy Configuration Log Rules

Increased under the frontend of/etc/haproxy/haproxy.conf

Option httplogoption logasaplog logserverip local5capture request header Host Len 40capture request Header X-forwarded-for Len 50#capture request header accept-language Len 50capture request header Referer Len 200capture request Header User-age NT Len 200

2.syslog configuration Enable remote receive

3.Logstash Configuration

Indexer

input {        file {                 path =>  "/var/log/haproxy.log"                  start_position  => beginning                 sincedb_write_interval => 0                 type =>  "Haproxy_log"                  codec => plain {                          charset =>  "Iso-8859-1"                  }        }}output {          #stdout  { codec => rubydebug}         redis {                 data_type =>  "List"                  key =>  "Logstash:haproxy_log"                  host =>  "192.168.1.2"                  port =>  6379        }}

Shipping

Input {    redis {        data_type  =>  "List"         key =>  "Logstash:haproxy_ Log "        host => " 192.168.1.2 "         port => 6379        threads  => 5        type =>  "HAPROXY_LOG"      }}filter {       grok{           match => ["message"  ,  "%{syslogtimestamp:syslog_timestamp}"  %{iporhost:syslog_server} %{syslogprog}: %{ip:client_ip}:%{int:client_port} \[%{monthday: Haproxy_monthday}/%{month:haproxy_month}/%{year:haproxy_year}:(?! &LT;[0-9])%{hour:haproxy_hour}:%{minute:haproxy_minute} (?::%{second:haProxy_second}) (?! [0-9]). %{int:haproxy_milliseconds}\] %{notspace:frontend_name} %{notspace:backend_name}/%{notspace:server_ name} %{int:time_request}/%{int:time_queue}/%{int:time_backend_connect}/%{int:time_backend_response}/\+%{ Notspace:time_duration} %{int:http_status_code} \+%{notspace:bytes_read} %{data:captured_ request_cookie} %{data:captured_response_cookie} %{notspace:termination_state} %{int:actconn}/% {int:feconn}/%{int:beconn}/%{int:srvconn}/%{notspace:retries} %{int:srv_queue}/%{int:backend_queue}  (\{%{iporhost:host}\|?) (%{ip:x_forward_for})? \|? (%{uri:referer})? \|%{greedydata:user_agent}\})? ( ) ( )? \ "(<badreq>| ( %{word:http_method}  (%{uriproto:http_proto}://)? (?:%{user:http_user} (?:: [^@]*) [email protected])? (?: %{urihost:http_host})? (?:%{uripathparam:http_request})? ( http/%{number:http_version})?)? \ ""]      }        useragent&nbsp {                source = >  "User_agent"                  target =>  "UA"         }         if [X_Forward_For] =~  "."  {           geoip {                 source => ["X_Forward_For "]                database  =>  "/usr/local/logstash2.2.2/bin/geolitecity.dat"            }        } else {            geoip {                source => [ "Client_ip"]                 database =>  "/usr/local/logstash2.2.2/bin/geolitecity.dat"            }        }        date{          match => ["Log_timestamp",   "Yyyy-mm-dd hh:mm:ss"  ]          timezone  => "Etc/uct"        }        Mutate{                remove_ field => ["Log_timestamp"]                 remove_field => [  "Host"  ]                 remove_field => [  "Path"  ]                 remove_field => [   "pid"  ]                 remove_field => [  "Client_port"  ]                 remove_field => [  "program"  ]                 remove_field = > [  "Haproxy_monthday"  ]                 remove_field => [  "Haproxy_month"  ]                 remove_field => [  "Haproxy_year"  ]                 remove_field = > [  "Haproxy_hour"  ]                 remove_field => [  "Haproxy_minute"  ]                 remove_field => [  " Haproxy_second " ]                 remove_field => [  "Haproxy_milliseconds"  ]                 remove_field => [  " Frontend_name " ]                 remove_field => [  "Captured_response_cookie"  ]                 remove_field => [  "Captured_request_cookie"  ]           convert => [  "Timetaken", "integer"  ]          convert => [  "Http_status _code "," integer " ]          convert => [   "Bytes_read", "integer"  ]          convert = > [  "Time_duration", "integer"  ]           convert => [  "Time_backend_response", "integer"  ]           convert => [  "Actconn", "integer"  ]           convert => [  "Feconn", "integer"  ]           convert => [  "Beconn", "integer"  ]           convert => [  "Srvconn", "integer"  ]           convert => [  "retries", "integer"  ]           convert => [  "Srv_queue", "integer" &NBSP;]           convert => [  "Backend_queue", " Integer " ]          convert => [ " Time_request "," integer " ]          convert =>  [  "Time_queue", "integer"  ]          convert  => [  "Time_backenD_connect "," integer " ]      }}output {          #stdout  { codec => rubydebug}         elasticsearch {                 hosts =>  "192.168.1.20:9200"                  index =>  "logstash-%{+yyyy. MM.DD} "&NBSP;&NBSP;&NBSP;&NBSP;&NBSP;&NBSP;&NBSP;&NBSP;}}

This article is from "Maple Night" blog, please be sure to keep this source http://fengwan.blog.51cto.com/508652/1755489

Elkstack Real-time analysis of Haproxy access log configuration