#整个配置文件分为三部分: Input,filter,output
#参考这里的介绍 https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html
Input {
#file可以多次使用, you can also write only one file and set its Path property to configure multiple files for multi-file monitoring
File {
#type是给结果增加了一个属性叫type值为 the entry for "<xxx>". The type here corresponds to the type in index in ES, that is, if you enter ES without specifying a type, the type here will be the type of index in ES.
Type = "Apache-access"
Path = "/apphome/ptc/windchill_10.0/apache/logs/access_log*"
#start_position可以设置为beginning或者end, beginning means to read the file from the beginning, and end to read the latest, which is also used with Ignore_older.
Start_position = Beginning
#sincedb_path表示文件读取进度的记录, each row represents a file, each line has two digits, the first represents the inode of the file, and the second represents the location to which the file was read (Byteoffset). Default is $home/.sincedb*
Sincedb_path = "/opt/logstash-2.3.1/sincedb_path/access_progress"
#ignore_older表示了针对多久的文件进行监控, the default day, in seconds, can be customized, such as by default, only read the modified files in a day.
Ignore_older = 604800
#add_field增加属性. This uses the ${hostname}, which is the native environment variable, if you want to use the local environment variables, you need to add--alow-env on the start command.
Add_field = {"Log_hostname" = "${hostname}"}
#这个值默认是 a newline character, if set to null "", the consequence is that each character represents an event
delimiter = ""
#这个表示关闭超过 (default) trace files after 3,600 seconds. This is especially useful for multiline. ... This parameter and Logstash on the way the file is read, two ways read tail, if it is read
Close_older = 3600
Coodec = Multiline {
Pattern = "^\s"
#这个negate是否定的意思, which means contrary to pattern, is not satisfying the meaning of patter.
# negate = ""
#what有两个值可选 previous and Next, for example, Java exceptions start with a space in the second line, where you can start with the pattern match space, what is set to previous means that the line begins with the previous row belongs to the same event. Another example is that sometimes a command is too long, and when the end of the line indicates that it belongs to the same event as the next line, you need to use negate=>true,what=> ' next '.
what = "Previous"
Auto_flush_interval = 60
}
}
File {
Type = "Methodserver-log"
Path = "/apphome/ptc/windchill_10.0/windchill/logs/methodserver-1604221021-32380.log"
Start_position = Beginning
Sincedb_path = "/opt/logstash-2.3.1/sincedb_path/methodserver_process"
# Ignore_older = 604800
}
}
filter{
#执行ruby程序, the following example converts a date to a string to give Daytag
Ruby {
Code = "event[' daytag ') = Event.timestamp.time.localtime.strftime ('%y-%m-%d ')"
}
# if [path] =~ "Access" {} else if [path] =~ "Methodserver" {} else if [path] =~ "Servermanager" {} else {} Note the statement structure
If [path] =~ "Methodserver" {#z这里的 =~ is a matching regular expression
Grok {
Patterns_dir = ["/opt/logstash-2.3.1/patterns"] #自定义正则匹配
# Tue 4/12/16 14:24:17:tp-processor2:hirecode---->77ls
Match + = {"Message" = "%{day:log_weekday}%{date_us:log_date}%{time:log_time}:%{greedydata:log_data}"}
}
#mutage是做转换用的
Mutate {
Replace + = {"Type" = "Apache"} #替换属性值
convert = {#类型转换
"Bytes" = "integer" #例如还有float
"duration" = "integer"
"state" = "integer"
}
#date主要是用来处理文件内容中的日期的. The content reads a string and converts it to @timestamp by date. Reference Https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html#plugins-filters-date-match
# date {
# match = ["LogTime", "Dd/mmm/yyyy:hh:mm:ss Z"]
# }
}else if [type] in [' Tbg_qas ', ' Mbg_pre '] {# if ... else if
}else {
drop{} # Discards the event
}
}
Output {
stdout{Codec=>rubydebug} # Direct output, debugging easy to use
# Output to Redis
Redis {
Host = ' 10.120.20.208 '
data_type = ' list '
Key = ' 10.99.201.34:access_log_2016-04 '
}
# Output to ES
Elasticsearch {
hosts = "192.168.0.15:9200"
index = "%{sysid}_%{type}"
Document_type = "%{daytag}"
}
}
