Linux Shell 網頁抓取

 args.txt

#! /bin/bash if [ -z $1 ] || [ ! -e $1 ]  thenecho "Usage: cmd.sh input "exitfiecho $0for num in $*;do     echo "$num"done for i in $(seq -3 $#);     do           echo $i     done for i in {0..5} do     echo $i done echo $@   for((i=4;i<7;i++));do echo $i done   echo "all:$$"  trimReg="s/\(^ *\)\(.*[^ ]\)\( *$\)/\2/"  tmpfile=`cat /proc/sys/kernel/random/uuid`     while read line; do     value=${line#*=}     key=${line%%=*}     key=`echo ${key}|sed -e "${trimReg}"`     value=`echo ${value}|sed -e "${trimReg}"`     if [ "$key" == "url" ]     then             url=$value     elif [ "$key" == "beginwith" ]     then             beginwith=$value     elif [ "$key" == "endwith" ]     then              endwith=$value     elif [ "$key" == "pagereg" ]     then              pagereg=$value     elif [ "$key" == "savepath" ]     then             savepath=$value  elif [ "$key" == "prefix" ] then   prefix=$valueelif [ "$key" == "proxy" ]thenproxy=$value    fi   done < $1  echo "url:$url"echo "beginwith:$beginwith"echo "pagereg:$pagereg"echo "endwith:$endwith"echo "prefix:$prefix"echo "proxy:$proxy"echo "savepath:$savepath"echo "tmpfile:$tmpfile"if [ -z $proxy ]thencontent=`curl -s $url | iconv -f gbk -t utf-8`elsecontent=`curl -x $proxy -s $url | iconv -f gbk -t utf-8`fi length=`expr length "${content}"` echo "download:$length byte(s)"content=${content#*${beginwith}} content=${content%%${endwith}*}length=`expr length "${content}"` echo "after filer:$length byte(s)"echo $content|grep -Po "$pagereg"|uniq > $savepathawk '{a[$0]++}END{for(m in a) print m}' $savepath > $tmpfileif [ ! -z $prefix ]thensed "s/^/$prefix/g" $tmpfile > $savepathelsecp $tmpfile $savepathfirm -f $tmpfile  str="0000012345456789000000"echo $str#str= expr substr $str 1 2 #str=${str:2:3} str=${str#*0} echo $str#trim the string str="  s =  "str=`echo $str | sed -e "${trimReg}"`  echo [$str] echo $str | sed -e "${trimReg}"

 

url = focus.news.163.combeginwith = <ul class="focuslist-1" id="focusTab-1">pagereg = (?<=href=\\")http://focus\\.news\\.163\\.com/[\\d]+.+?(?=\\")endwith =  <div class="con-4" area clearfix">savepath = 163.txt