LOGSTASH/CONF.D File Preparation

Logstash-01.conf

Input {

Beats {
Port = 5044
Host = "0.0.0.0"
Type = "Logs"
codec = "JSON"
}

}

filter{

if ([type] = = "Nginx-access") {

Grok {
Match + = {
"Request" = "\s+" (? <api_path>.+?) (\?. *)? \s+ "
}
}
Grok {
Match + = {
"Agent" = "(? <browser>maxthon| qqbrowser| chrome| safari| Firefox| opera| MSIE?) (/[0-9.) +)?"
}
}
Grok {
Match + = {
"Agent" = "(? <os>macintosh| android| symbianos|ipad|iphone|ipod| Linux| Windows?) "
}
}
Mutate {
split = ["Upstreamtime", ","]
}

Mutate {
Remove_field = ["offset", "@version", "Beat", "Input_type", "tags", "id"]
}

Date {
match = ["timestamp", "Dd/mmm/yyyy:hh:mm:ss Z"]
}

geoip{
Source = "ClientIP" # taken from client IP
target = "GeoIP"
in Nginx "/usr/share/logstash/g Eoip/geolite2-city.mmdb "
}
Mutate {
Convert = = [" [Geoip][location] "," float "]
convert = = [" [Respo Nsetime] "," float "]
convert = = [" [Upstreamtime] "," float "]
convert = = [" [Size] "," float "]
convert = > ["[Status]", "float"]
convert = = ["[Cookie]", "float"]
}

}
if ([type] = = "Nginx-error") {
Grok {
Match = = ["Message", "(? <timestamp>%{year}[./-]%{monthnum}[./-]%{monthday}[-]%{time}) \[%{l Oglevel:severity}\]%{posint:pid}#%{number}:%{greedydata:errormessage} (?:, Client: (? <remote_addr>%{ip}|%{ HOSTNAME})) (?:, Server:%{iporhost:server}?) (?:, Request:%{qs:request})? (?:, Upstream: (? <upstream>\ "%{uri}\" |%{qs}))? (?:, Host:%{qs:request_host})? (?:, referrer: \ "%{uri:referrer}\")? "]
}
}

}
Output {

stdout {
codec = rubydebug {}
}

if [type] = = "Nginx-access" {
#redis {
# host = "127.0.0.1"
# port = "6379"
# password = ' 123.com '
# db = ' 2 '
# data_type = "List"
# key = ' nginx-access '
# }

Elasticsearch {
hosts = ["192.168.10.196:9200"]
index = "logstash-nginx-access-%{+yyyy. MM.DD} "
#codec = "JSON"
Template_overwrite = True
}
}
if [type] = = "Nginx-error" {

Elasticsearch {
hosts = ["192.168.10.196:9200"]
index = "logstash-nginx-error-%{+yyyy. MM.DD} "
codec = "JSON"
Template_overwrite = True
}
}
}

Logstash-java-02.conf

Input {
Redis {
Host = "192.168.10.196"
Port = 6379
Password = "123.com"
db = "1"
data_type = "List"
Key = "Java-access"
codec = "JSON"
Threads = 5
}
stdin{
Codec = Multiline {
#pattern = "^%{loglevel}"
Pattern = "^%{timestamp_iso8601}"
Negate = True
what = "Previous"
}
}

}

filter{

if ([type] = = "Java-access") {
grok{
Match + = {"Message" = "%{timestamp_iso8601:timestamp}%{loglevel:level}%{javalogmessage:msg}"}
}

Date {
Match = ["timestamp", "Yyyy-mm-dd hh:mm:ss,s", "ISO8601"]
}
JSON {
Source = "message"
}
}
}
Output {

stdout {
codec = rubydebug {}
}

if [type] = = "Java-access" {
Elasticsearch {
hosts = ["192.168.10.196:9200"]
index = "logstash-java-access-%{+yyyy. MM.DD} "
Template_overwrite = True
}
}
}

Logstash-oms-04.conf

Input {
Redis {
Host = "192.168.10.196"
Port = 6379
db = "0"
data_type = "List"
Password = "123.com"
Key = "Oms-access"
codec = "JSON"
Threads = 5
}

}

filter{
if ([type] = = "Java-oms") {
grok{
Match + = {"Message" = "%{timestamp_iso8601:timestamp}%{loglevel:level}%{javalogmessage:msg}"}
}

Date {
Match = ["timestamp", "Yyyy-mm-dd hh:mm:ss,s", "ISO8601"]
}
JSON {
Source = "message"
}
}

}
Output {

stdout {
codec = rubydebug {}
}

if [type] = = "Java-oms" {
Elasticsearch {
hosts = ["192.168.10.196:9200"]
index = "logstash-java-oms-%{+yyyy. MM.DD} "
Template_overwrite = True
}
}
}

Logstash-payment-03.conf

Input {
Redis {
Host = "192.168.10.196"
Port = 6379
db = "1"
data_type = "List"
Password = "123.com"
Key = "Java-access"
codec = "JSON"
Threads = 5
}

}

filter{
if ([type] = = "Java-payment") {
grok{
Match + = {"Message" = "%{timestamp_iso8601:timestamp}%{loglevel:level}%{javalogmessage:msg}"}
}

Date {
Match = ["timestamp", "Yyyy-mm-dd hh:mm:ss,s", "ISO8601"]
}
JSON {
Source = "message"
}
}
}
Output {

stdout {
codec = rubydebug {}
}

if [type] = = "Java-payment" {
Elasticsearch {
hosts = ["192.168.10.196:9200"]
index = "logstash-java-payment-%{+yyyy. MM.DD} "
Template_overwrite = True
}
}
}

LOGSTASH/CONF.D File Preparation