Shell script: Nginx Access log cutting and packaging upload

Scripting Features:

Log cutting class script, including: Nginx access logs, gateway devices, AC devices, authentication information and so on, all through FTP upload to the appropriate server to save.

---------------

#!/bin/bash# cut and upload nginxlog# by colin# revision on  2016-02-22#########################################  function Description: This script applies to cutting and uploading nginxlog##  update instructions: ################ ######################### #sleep  60    #延时60秒运行PATH =/usr/local/sbin:/usr/local/bin:/sbin:/ bin:/usr/sbin:/usr/binrun_log= '/var/log/cron_scripts_run.log ' [ ! -f ${run_log} ]  && touch ${RUN_LOG}echoGoodLog  () {    echo -e  "\033[32m" Date +%f " "%T ":"%N '  $*\033[0m " >> ${RUN_LOG}}echoBadLog  () {     echo -e  "\033[31m ' date +%f"   "%T": "%N '  $*\033[0m"  >> ${run _log}}echogoodlog  "now, script:  ' basename $0 '  run." runlog_max_num=100000runlog_max_delnum=5000send_max_time=6send_while_sleep_time=30back_save_max_day=180        #  log Local save days   #LOg_cut_min=60   #  log file 30 minutes cut once   #cleanRunLog   () {     Cleanlogfile=${1? " usage:  $FUNCNAME  log_file_name "}    temp_wc=$ ( wc -l < ${ cleanlogfile} )     [  "${TEMP_WC}"  -gt  "${runlog_max_num}"  ]  && {        sed -i  "1,${runlog_max_delnum}d"  ${CLEANLOGFILE} && echoGoodLog  "Clean up the ${cleanlogfile} ..."     }    echoGoodLog  "script:  ' basename $0 '  run  done. "     exit}server_name=${1? Usage: $0 jnweb1 /nginxlog/sd/jn/} cd_dir=$2logs_path= "/data/store/logs/www/" LOG_NAME= "m_ Wonaonao_access "log_type=" Nginx "log_typeb=" Nginx "lcd_dir="/data/store/logs/backup/"station=" echo $ ( hostname)  |awk -F-  ' {print $3} ' |tr [a-z]  [A-z] ' ########################################### define the ftp server#ftpserver= ' IP address ' Ftpuser= ' username ' ftppasswd= ' password ' sshport= ' port ' ####################################### #NGINX_PID = ' cat /var/run/ Nginx.pid ' t= ' echo $ (date +%k) |sed  ' s/ //g ' last_hour_time= ' date +%y-%m-%d-%h ' FILENUM= ' Expr $ (date +%m|sed  ' s/^0//g ')  / ${log_cut_min} ' record_log_value= ' ${FILENUM} ' [  "${filenum}"  -eq 0 ] && {    filenum= ' expr 60 /  ${log_cut_min} '     last_hour_time= ' date -d  "1 hour ago"  +% Y-%m-%d-%h '}last_t_time= ' date -d  "1 hour ago"  +%h ' [  "${RECORD_LOG_VALUE}"  - Gt 0 ] && last_t_time= ' date +%h ' last_date_time= ' date -d  "Yesterday"  +%y-%m-%d ' restartnginx  () {    /usr/sbin/service nginx restart  && echogoodlog  "Restart nginx is done."  | |  {        echoBadLog  "restart nginx is failed ,  please check ... "    }}restartphp () {    ##     # apt-get installation is not the same as the startup script name for the source installation     #    /usr/sbin/service  php5-fpm restart | |  /usr/sbin/service php-fpm restart    if [ $? -eq 0  ];then        echoGoodLog  "Restart php5-fpm is  done. "     else        echoBadLog  "restart  Php5-fpm is failed, please check "    fi}###  to be compatible with older versions, when cut in 60 minutes, Just like the old version file name. #if [ ${log_cut_min} -eq 60 ];then    tar_log_hour_ Name= "${last_hour_time}.${sErver_name}.nginxlog.tar.gz "Else    tar_log_hour_name=" ${last_hour_time}-${filenum}.${ Server_name}.nginxlog.tar.gz "Ficd ${lcd_dir} && [ ! -e ${tar_log_hour_ name} ] && {    ##    #  Backup M_wonaonao_ Access log     #    [ -e ${logs_path}${log_name}.log ]  && mv ${logs_path}${log_name}.log ${log_name}_${last_t_time}.log | |  {        echoBadLog  "Log: ${logs_path}${log_name}.log  is not exist ... "        restartNginx         restartphp    }    [ -e ${log_ name}_${last_t_time}.log -a -s ${log_name}_${last_t_time}.log ] && {       &nbSp; kill -usr1 ${nginx_pid}        cat ${log_name} _${last_t_time}.log >> ${log_name}.log | |  cat ${LOG_NAME}_${LAST_T_TIME}.log >> ${LOG_NAME}.log         check_log= ' stat ${log_name}.log |grep  ' Modify: ' |awk  ' {print $3} ' | awk -f:  ' {print $1} '  |sed  ' s/^0//g '          if [  "${check_log}"  -eq  "${t}"  ];then             echoGoodLog  "Log: ${log_type}_log is cut successfully ..."         else             echoBadLog  "Cut: ${log_type}_log was failed, please check ..."         fi   &nBSP;}  | |  echoBadLog  "Log: ${log_name}_${last_t_time}.log is null."     ##    #  backup M_wonaonao_record logs, package     #     [ -e ${LOGS_PATH}m_wonaonao_record_${LAST_T_TIME}.log ] &&  {        if [ ${record_log_value} -gt 0  ];then            cp -a ${logs_ path}m_wonaonao_record_${last_t_time}.log  .          Else            mv ${logs_path}m_wonaonao_ record_${last_t_time}.log  .         fi         tar -zcf ${tar_log_hour_name} ${log_name}_${last_t_time}. Log m_wonaonao_record_${last_t_time}.log    } | |  tar -zcf ${tar_log_hour_name} ${log_name}_${last_t_time}.log    if  [ -e ${TAR_LOG_HOUR_NAME} ];then         echogoodlog  "Tar: ${tar_log_hour_name} is successfully in every hour ..."     else        echoBadLog  "Tar: ${log_ Type}_log of every hour was failed, please check ... "     fi}tar_log_day_name= "${last_date_time}.${server_name}.nginxlog.tar.gz" tardaylog  () {     tar_day_log=$1    cd ${lcd_dir} && [ -e ${tar _day_log} ] | |  {    tar -zcpf ${tar_day_log} --remove-files ${log_name}.log  m_wonaonao_record_*.log    if [ -e ${tar_day_log} ]; then        echogoodlog  "tar: ${tar_day_log} is  Successfully in every day ... "    else         echoBadLog  "tar: ${tar_day_log} of the whole day was  Failed, please check "    fi    }}###  packed daily full log #[   "${t}"  -eq 0 -a  "${record_log_value}"  -eq 0 ] &&  Tardaylog ${tar_log_day_name}ftp_log_dir= "/tmp/ftp_err" [ -d ${ftp_log_dir} ] | |  mkdir -p ${ftp_log_dir}ftp_error_log= "${ftp_log_dir}/ftp_temp_${log_type}_err$$.log" SendLog   () {    sendlogfile=$1    ftp -ivn ${ftpserver} 21  >${ftp_error_log} << _eof_user ${ftpuser} ${ftppasswd}passivebinlcd ${ Lcd_dir}cd  ${cd_dir}put ${sendlogfile}bye_eof_    ##    #  count the number of error logging lines in front of the FTP run output     #    log_count= ' grep -w  "^226" &NBSP;${FTP_ERROR_LOG}|WC  -l '     if [  ' ${log_count} '  -eq 1 ];then         echoGoodLog  "Send: ${sendlogfile} to ftp_server was  successfully. "         TEMP_SEND_STATUS=0         return 0    else        echobadlog   "send: ${sendlogfile} more than  $x  time."         TEMP_SEND_STATUS=1         sleep ${SEND_WHILE_SLEEP_TIME}        return 1     fi}reuploadlist= "/var/log/reuPload_list_${log_typeb}_${station}.log "temp_reuploadlist="/var/log/temp_reupload_list_${log_typeb}_${station}. Log "[ -f ${temp_reuploadlist} ] && rm ${temp_reuploadlist}runsendlog   () {    sendlogname=$1    x=1;i=1    until  [  "$i" &NBSP;-EQ&NBSP;0&NBSP;];d o        [  "$x"  - gt  "${send_max_time}"  ] && {             echoBadLog  "send: ${sendlogname} to ftp_server was failed,  please check ... "            echo " ${lcd_dir};; ${cd_dir};; ${sendlogfile} " >> ${TEMP_REUPLOADLIST}             break        }      &nBsp; sendlog ${sendlogname}        i= ' echo $? '          x= ' expr  $x  + 1 '     done}[  "$ {T} " -eq 0 -a -e " ${lcd_dir}${tar_log_day_name} " -a " ${record_log_value} "  -eq 0 ] && runSendLog ${TAR_LOG_DAY_NAME}[ -e  "${lcd_dir}${ Tar_log_hour_name} " ] && runSendLog ${TAR_LOG_HOUR_NAME}###  Update the above two successful records to this temporary file #temp_sendsuccesfile= "/var/log/temp_send_succes_${log_typeb}_${station}.txt" Sendsuccesfile= "/var/log/send_succes_${log_typeb}_${station}.txt" [  "${temp_send_status}"  -eq 0  ] && echo  "${lcd_dir};; ${cd_dir};; ${sendlogfile};; $ (date +%s) " > ${TEMP_SENDSUCCESFILE}###  function: After power-off recovery, send a full day log file on the day of power loss #[ -e ${ Sendsuccesfile} ] && {    lcd_dir= ' CAT&NBSp;${sendsuccesfile}|awk -f ";;"   ' {print $1} '     cd_dir= ' cat ${sendsuccesfile}|awk -f ";;"   ' {print $2} '     last_date_success= ' date -d @ ' $ (cat ${ Sendsuccesfile}|awk -f ";;"   ' {print $4} ') " +%y-%m-%d '     interval_time= ' expr $ (date +%s)  - $ (cat ${sendsuccesfile}|awk -f ";;"   ' {print $4} ') '     last_day= ' echo $ (date -d @ "$ (cat ${ Sendsuccesfile}|awk -f ";;"   ' {print $4} ') '  +%d ' |sed  ' s/^0//g '     now_day= ' echo $ (date  +%d) |sed  ' s/^0//g '     ##    # 7000, this value must be greater than 3600 and less than 7200     #+  tested, when equal to 7000, the result of the operation is in line with the expected     #    [  " ${last_day} " -ne " ${now_day} " -a " ${interval_time} " -gt 7000 ] && Amp; {        cd ${lcd_dir} && {             temp_send_files= "${LAST_DATE_SUCCESS}.${SERVER_ Name}.nginxlog.tar.gz "            [ -e $ {temp_send_files} ] | |  tarDayLog ${TEMP_SEND_FILES}        }         runSendLog ${TEMP_SEND_FILES}    }     rm ${sendsuccesfile}}[ -f ${temp_sendsuccesfile} ] && mv ${ temp_sendsuccesfile} ${sendsuccesfile}###  retransmission of the file that failed last send #reuploadfile  () {     temp_need_do_file=$1    reuploadlist_num=$ ( wc -l < ${temp_need_do_ file} )     [  "${reuploadlist_num}"  -ge 1 ] && {        while read line         do            lcd_dir= ' Echo ${line}|awk -f ";;"   ' {print $1} '             cd_dir= ' echo  ${line}|awk -f ";;"   ' {print $2} '              Reuploadfilename= ' echo ${line}|awk -f ";;"   ' {print $3} '             [ -f   "${lcd_dir}/${reuploadfilename}"  ] && runSendLog ${REUPLOADFILENAME}         done < ${temp_need_do_file}    }     [ -e ${temp_need_do_file} ] && rm ${temp_need_do _file}}[ -s ${reuploadlist}&nbsP;]  && reUploadFile ${REUPLOADLIST}[ -f ${TEMP_REUPLOADLIST} ] & & mv ${TEMP_REUPLOADLIST} ${REUPLOADLIST}[ -f ${FTP_ERROR_LOG} ] & & rm ${FTP_ERROR_LOG}###  Clean up the backup log #[ -d ${lcd_dir} ] && cd  ${LCD_DIR} && {    for FILENAME in  ' find .  -type f -ctime +${BACK_SAVE_MAX_DAY} | awk -F/  ' {print $2} '     do        rm  ${FILENAME} & & echoGoodLog  "Clear: ${lcd_dir}/${filename} ..."     done}cleanrunlog  ${run_log}

---------------

This article from "Mountain Road 18 Bend" blog, reproduced please contact the author!

Shell script: Nginx Access log cutting and packaging upload