Use shell and redis to collect and analyze logs in cluster services in real time

 

650) this. width = 650; "border =" 0 "alt =" "src =" http://www.bkjia.com/uploads/allimg/131227/1T1113S9-0.png "/>

In the statistical project, the most difficult to implement is the collection of log data. Logs are distributed in various data centers across the country, and the data volume is large. Such methods as rsync + inotify obviously cannot meet the requirements of fast log synchronization. Of course, you can also use fluentd and flume to collect log data. In addition to this, you can also write a simple set.

 

The process of writing this log analysis system is:

 
 

  
  
1. Collect data on the client and send the data to the server through redis pub.
  
  
 
  
  
2. The server is a redis sub, which stores data in a single file, or filters out the data currently.
 
 

 

Update Data of logs collected by the client

 
 

  
  
#! /Bin/bash
  
  
 
  
  
DATE = 'date + % s'
  
  
LOGFILE = $1
  
  
If [! -F $1]; then
  
  
Echo "LOG file did not give or it's not a file"
  
  
Fi
  
  
Sleep_time = "2"
  
  
 
  
  
Count_init = 'wc-l $ {LOGFILE} | awk '{print $1 }''
  
  
While true
  
  
Do
  
  
DATE_NEW = 'date + % s'
  
  
# DATE = $ (date + % s)
  
  
Count_new = 'wc-l $ {LOGFILE} | awk '{print $1 }''
  
  
Add_count = $ ($ {count_new}-$ {count_init }))
  
  
Count_init =$ {count_new}
  
  
If [! -N "$ {add_count}"]
  
  
Then
  
  
Add_count = 0
  
  
Fi
  
  
QPS = $ ($ {add_count}/$ {sleep_time }))
  
  
Info = 'Tail-n $ {add_count }$ {LOGFILE }'
  
  
Echo $ info
  
  
# We can pass out the info value.
  
  
Echo "Then QPS at 'date-d" 1970-01-01 UTC $ {DATE_NEW} seconds "+" % Y-% m-% d % H: % M: % S "'is" $ {QPS}
  
  
# Echo "DATE_NEW:" $ DATE_NEW "DATE_PLUS:" $ DATE_PLUS
  
  
Sleep $ sleep_time
  
  
Done
 
 

 

650) this. width = 650; "border =" 0 "alt =" "src =" http://www.bkjia.com/uploads/allimg/131227/1T111AH-1.png "/>

Print real-time logs

650) this. width = 650; "border =" 0 "alt =" "src =" http://www.bkjia.com/uploads/allimg/131227/1T1116350-2.jpg "/>

 

To upload the command to the server, you only need to add the following command in the script to get it done ~

/Root/redis-bash-cli-h 10.10.10.61 PUBLISH rui "$ info"

 

Redis-bash-cli this is a client script that can pass the data publish

 
 

  
  
#!/bin/bash   
  
  
source /usr/share/redis-bash/redis-bash-lib 2> /dev/null  
  
  
if [ $? -ne 0 ]; then  
  
  
  LIBFOLDER=${0%/${0##*/}}  
  
  
  source ${LIBFOLDER}/redis-bash-lib 2> /dev/null  
  
  
  if [ $? -ne 0 ]; then  
  
  
    echo "can't find redis-bash-lib in /usr/share/redis-bash or ${LIBFOLDER}"  
  
  
    exit 127   
  
  
  fi  
  
  
fi  
  
  
REDISHOST=localhost  
  
  
REDISPORT=6379  
  
  
REPEAT=1  
  
  
DELAY=0  
  
  
while getopts ":h:n:p:r:a:i:" opt  
  
  
do  
  
  
    case ${opt} in  
  
  
        h) REDISHOST=${OPTARG};;  
  
  
        n) REDISDB=${OPTARG};;  
  
  
        p) REDISPORT=${OPTARG};;  
  
  
        r) REPEAT=${OPTARG};;  
  
  
        a) AUTH=${OPTARG};;  
  
  
        i) DELAY=${OPTARG};;  
  
  
    esac  
  
  
done  
  
  
shift $((${OPTIND} - 1))  
  
  
if [ "${REDISHOST}" != "" ] && [ "${REDISPORT}" != "" ]  
  
  
then  
  
  
    exec 6<>/dev/tcp/${REDISHOST}/${REDISPORT} # open fd  
  
  
    if [ $? -ne 0 ]; then  
  
  
        exit 1  
  
  
    fi  
  
  
else  
  
  
    echo "Wrong arguments"  
  
  
    exit 255  
  
  
fi  
  
  
[ "${AUTH}" != "" ] && redis-client 6 AUTH ${AUTH} > /dev/null  
  
  
[ "${REDISDB}" != "" ] && redis-client 6 SELECT ${REDISDB} > /dev/null  
  
  
for ((z=1;z<=${REPEAT};z++))  
  
  
do  
  
  
    redis-client 6 "${@}"  
  
  
    if [ $? -ne 0 ]; then  
  
  
        exit 1  
  
  
    fi  
  
  
    [ ${DELAY} -gt 0 ] && sleep ${DELAY}  
  
  
done  
  
  
exec 6>&- #close fd  
 
 

 

Log Service

650) this. width = 650; "border =" 0 "alt =" "src =" http://www.bkjia.com/uploads/allimg/131227/1T1115196-3.png "/>

 

Redis-publish-test: This is the log server and can receive publish data.

 

 
 

  
  
#!/bin/bash   
  
  
source /usr/share/redis-bash/redis-bash-lib 2> /dev/null  
  
  
if [ $? -ne 0 ]; then  
  
  
  LIBFOLDER=${0%/${0##*/}}  
  
  
  echo $LIBFOLDER  
  
  
  source ${LIBFOLDER}/redis-bash-lib 2> /dev/null  
  
  
  if [ $? -ne 0 ]; then  
  
  
    echo "can't find redis-bash-lib in /usr/share/redis-bash or ${LIBFOLDER}"  
  
  
    exit 127   
  
  
  fi  
  
  
fi  
  
  
REDISHOST=localhost  
  
  
REDISPORT=6379  
  
  
while getopts ":h:p:" opt  
  
  
do  
  
  
    case ${opt} in  
  
  
        h) REDISHOST=${OPTARG};;  
  
  
        p) REDISPORT=${OPTARG};;  
  
  
    esac  
  
  
done  
  
  
shift $((${OPTIND} - 1))  
  
  
while true  
  
  
do  
  
  
    exec 5>&-  
  
  
    if [ "${REDISHOST}" != "" ] && [ "${REDISPORT}" != "" ]  
  
  
    then  
  
  
        exec 5<>/dev/tcp/${REDISHOST}/${REDISPORT} # open fd  
  
  
    else  
  
  
        echo "Wrong arguments"  
  
  
        exit 255  
  
  
    fi  
  
  
    redis-client 5 SUBSCRIBE ${1} > /dev/null # subscribe to the pubsub channel in fd 5  
  
  
    while true  
  
  
    do  
  
  
        unset ARGV  
  
  
        OFS=${IFS};IFS=$'\n' # split the return correctly  
  
  
        ARGV=($(redis-client 5))  
  
  
        IFS=${OFS}  
  
  
        if [ "${ARGV[0]}" = "message" ] && [ "${ARGV[1]}" = "${1}" ]  
  
  
        then  
  
  
        echo ${ARGV[2]}  
  
  
        a=${ARGV[2]}  
  
  
            echo $($a)  
  
  
            echo "Message from pubsub channel: ${ARGV[2]}"  
  
  
        elif [ -z ${ARGV} ]  
  
  
        then  
  
  
            sleep 1  
  
  
            break  
  
  
        fi  
  
  
    done  
  
  
done  
 
 

 

 

This article is from "Fengyun, it's her ." Blog, please be sure to keep this source http://rfyiamcool.blog.51cto.com/1030776/1191926