Delete the spider record from the nginx log

Source: Internet
Author: User
#!/bin/shif [ -r $1 ]; then    #Delete Baiduspider    baidu=$(grep -c ‘Baiduspider‘ $1)    if [ $baidu -gt 0 ]; then        sed -i ‘/Baiduspider/d‘ $1        echo "match Baiduspider $baidu line."    else        echo "Baiduspider not found."    fi    #Delete YandexBot    yandex=$(grep -c ‘YandexBot‘ $1)    if [ $yandex -gt 0 ]; then        sed -i ‘/YandexBot/d‘ $1        echo "match YandexBot $yandex line."    else        echo "YandexBot not found."    fi    #Delete bingbot    bing=$(grep -c ‘bingbot‘ $1)    if [ $bing -gt 0 ]; then        sed -i ‘/bingbot/d‘ $1        echo "match bingbot $bing line."    else        echo "bingbot not found."    fi    #Delete EasouSpider    easou=$(grep -c ‘EasouSpider‘ $1)    if [ $easou -gt 0 ]; then        sed -i ‘/EasouSpider/d‘ $1        echo "match EasouSpider $easou line."    else        echo "EasouSpider not found."    fi    #Delete Googlebot    Googlebot=$(grep -c ‘Googlebot‘ $1)    if [ $Googlebot -gt 0 ]; then        sed -i ‘/Googlebot/d‘ $1        echo "match Googlebot $Googlebot line."    else        echo "Googlebot not found."    fi    #Delete Mediapartners-Google    mgoogle=$(grep -c ‘Mediapartners-Google‘ $1)    if [ $mgoogle -gt 0 ]; then        sed -i ‘/Mediapartners-Google/d‘ $1        echo "match Mediapartners-Google $mgoogle line."    else        echo "Mediapartners-Google not found."    fi    #Delete Yahoo    yahoo=$(grep -c ‘Yahoo‘ $1)    if [ $yahoo -gt 0 ]; then        sed -i ‘/Yahoo/d‘ $1        echo "match Yahoo $yahoo line."    else        echo "Yahoo not found."    fi    #Delete YoudaoBot    youdao=$(grep -c ‘YoudaoBot‘ $1)    if [ $youdao -gt 0 ]; then        sed -i ‘/YoudaoBot/d‘ $1        echo "match YoudaoBot $youdao line."    else        echo "YoudaoBot not found."    fi    #Delete XoviBot    xovi=$(grep -c ‘XoviBot‘ $1)    if [ $xovi -gt 0 ]; then        sed -i ‘/XoviBot/d‘ $1        echo "match XoviBot $xovi line."    else        echo "XoviBot not found."    fi    #Delete MSNBot    msn=$(grep -c ‘MSNBot‘ $1)    if [ $msn -gt 0 ]; then        sed -i ‘/MSNBot/d‘ $1        echo "match MSNBot $msn line."    else        echo "MSNBot not found."    fi    #Delete Sogou web spider    sogou=$(grep -c ‘Sogou web spider‘ $1)    if [ $sogou -gt 0 ]; then        sed -i ‘/Sogou web spider/d‘ $1        echo "match Sogou web spider $sogou line."    else        echo "Sogou web spider not found."    fi    #Delete JikeSpider    jike=$(grep -c ‘JikeSpider‘ $1)    if [ $jike -gt 0 ]; then        sed -i ‘/JikeSpider/d‘ $1        echo "match JikeSpider $jike line."    else        echo "JikeSpider not found."    fi    #Delete proximic    proximic=$(grep -c ‘proximic‘ $1)    if [ $proximic -gt 0 ]; then        sed -i ‘/proximic/d‘ $1        echo "match proximic $proximic line."    else        echo "proximic not found."    fi    #Delete 360Spider    so360=$(grep -c ‘360Spider‘ $1)    if [ $so360 -gt 0 ]; then        sed -i ‘/360Spider/d‘ $1        echo "match 360Spider $so360 line."    else        echo "360Spider not found."    fi    #Delete 360spider-image    so360img=$(grep -c ‘360spider-image‘ $1)    if [ $so360img -gt 0 ]; then        sed -i ‘/360spider-image/d‘ $1        echo "match 360spider-image $so360img line."    else        echo "360spider-image not found."    fi    #Delete YYSpider    yy=$(grep -c ‘YYSpider‘ $1)    if [ $yy -gt 0 ]; then        sed -i ‘/YYSpider/d‘ $1        echo "match YYSpider $yy line."    else        echo "YYSpider not found."    fi    #Delete AhrefsBot    ahrefs=$(grep -c ‘AhrefsBot‘ $1)    if [ $ahrefs -gt 0 ]; then        sed -i ‘/AhrefsBot/d‘ $1        echo "match AhrefsBot $ahrefs line."    else        echo "AhrefsBot not found."    fi    #Delete msnbot    msnbot=$(grep -c ‘msnbot‘ $1)    if [ $msnbot -gt 0 ]; then        sed -i ‘/msnbot/d‘ $1        echo "match msnbot $msnbot line."    else        echo "msnbot not found."    fi    #Delete YisouSpider    yisou=$(grep -c ‘YisouSpider‘ $1)    if [ $yisou -gt 0 ]; then        sed -i ‘/YisouSpider/d‘ $1        echo "match YisouSpider $yisou line."    else        echo "YisouSpider not found."    fi    #Delete Facebot    facebot=$(grep -c ‘Facebot‘ $1)    if [ $facebot -gt 0 ]; then        sed -i ‘/Facebot/d‘ $1        echo "match Facebot $facebot line."    else        echo "Facebot not found."    fi    #Delete GrapeshotCrawler    grapeshot=$(grep -c ‘GrapeshotCrawler‘ $1)    if [ $grapeshot -gt 0 ]; then        sed -i ‘/GrapeshotCrawler/d‘ $1        echo "match GrapeshotCrawler $grapeshot line."    else        echo "GrapeshotCrawler not found."    fi    #Delete WeSEE    wesee=$(grep -c ‘WeSEE‘ $1)    if [ $wesee -gt 0 ]; then        sed -i ‘/WeSEE/d‘ $1        echo "match WeSEE $wesee line."    else        echo "WeSEE not found."    fi    #Delete Sogou inst spider    sogou_inst=$(grep -c ‘Sogou inst spider‘ $1)    if [ $sogou_inst -gt 0 ]; then        sed -i ‘/Sogou inst spider/d‘ $1        echo "match Sogou inst spider $sogou_inst line."    else        echo "Sogou inst spider not found."    fi    #Delete WebZIP/x.x    webzip=$(grep -c ‘www.spidersoft.com‘ $1)    if [ $webzip -gt 0 ]; then        sed -i ‘/www.spidersoft.com/d‘ $1        echo "match WebZIP/x.x $webzip line."    else        echo "WebZIP/x.x not found."    fi    #Delete baidu mobile spider    bms=$(grep -c ‘www.baidu.com\/search\/spider.html‘ $1)    if [ $bms -gt 0 ]; then        sed -i ‘/www.baidu.com\/search\/spider.html/d‘ $1        echo "match baidu mobile spider $bms line."    else        echo "baidu mobile spider not found."    fi    #Delete ChinasoSpider    chinaso=$(grep -c ‘ChinasoSpider‘ $1)    if [ $chinaso -gt 0 ]; then        sed -i ‘/ChinasoSpider/d‘ $1        echo "match ChinasoSpider $chinaso line."    else        echo "ChinasoSpider not found."    fi    #Delete www.xxx.com    xxx=$(grep -c ‘www.xxx.com‘ $1)    if [ $xxx -gt 0 ]; then        sed -i ‘/www.xxx.com/d‘ $1        echo "match www.xxx.com $xxx line."    else        echo "www.xxx.com not found."    fi    #Delete SEOENGWorldBot    seoeng=$(grep -c ‘SEOENGWorldBot‘ $1)    if [ $seoeng -gt 0 ]; then        sed -i ‘/SEOENGWorldBot/d‘ $1        echo "match SEOENGWorldBot $seoeng line."    else        echo "SEOENGWorldBot not found."    fi    #Delete Mail.RU_Bot    mailru=$(grep -c ‘Mail.RU_Bot‘ $1)    if [ $mailru -gt 0 ]; then        sed -i ‘/Mail.RU_Bot/d‘ $1        echo "match Mail.RU_Bot $mailru line."    else        echo "Mail.RU_Bot not found."    fi    #Delete Girafabot    gira=$(grep -c ‘Girafabot‘ $1)    if [ $gira -gt 0 ]; then        sed -i ‘/Girafabot/d‘ $1        echo "match Girafabot $gira line."    else        echo "Girafabot not found."    fi    #Delete DotBot    dotBot=$(grep -c ‘DotBot‘ $1)    if [ $dotBot -gt 0 ]; then        sed -i ‘/DotBot/d‘ $1        echo "match DotBot $dotBot line."    else        echo "DotBot not found."    fi    #Delete BLEXBot    BLEXBot=$(grep -c ‘BLEXBot‘ $1)    if [ $BLEXBot -gt 0 ]; then        sed -i ‘/BLEXBot/d‘ $1        echo "match BLEXBot $BLEXBot line."    else        echo "BLEXBot not found."    fielse    echo "please, input your log file"fi

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.