一個python爬蟲工具類

標籤：ice   add   cat   utf8   參數   ons   dig   decorator   exception   

寫了一個爬蟲工具類。

# -*- coding: utf-8 -*-# @Time    : 2018/8/7 16:29# @Author  : cxa# @File    : utils.py# @Software: PyCharmfrom retrying import retryfrom decorators.decorators import decoratorfrom glom import glomfrom config import headersimport datetimeimport hashlib@retry(stop_max_attempt_number=3, wait_fixed=2000, stop_max_delay=10000)@decoratordef post_html(session,post_url:int,post_data:dict,headers=headers,timeout=30):    ‘‘‘    :param session: 傳入session對象    :param post_url: post請求需要的url    :param headers: 前序資訊,config模組預設提供    :param post_data: post資訊 字典類型    :param timeout:    :return:    ‘‘‘    post_req=session.post(url=post_url,headers=headers,data=post_data,timeout=timeout)    if post_req.status_code==200:        post_req.encoding=post_req.apparent_encoding        return post_req@retry(stop_max_attempt_number=3,wait_fixed=2000, stop_max_delay=10000)@decoratordef get_response(session,url:str,headers=headers,timeout=30):    ‘‘‘    :param url:    :return: return response object    ‘‘‘    req=session.get(url=url,headers=headers,timeout=timeout)    if req.status_code==200:        req.encoding=req.apparent_encoding        return req@decoratordef get_html(req):    source=req.text    return source@decoratordef get_json(req):    jsonstr=req.json()    return jsonstr@decoratordef get_xpath(req,xpathstr:str):    ‘‘‘    :param req:    :param xpathstr:    :return:    ‘‘‘    node=req.html.xpath(xpathstr)    return node@decoratordef get_json_data(jsonstr:str,pat:str):    ‘‘‘    #通過glom模組操作資料    :param jsonstr:    :param pat:    :return:    ‘‘‘    item=glom(jsonstr,pat)    return item@decoratordef get_hash_code(key):    value=hashlib.md5(key.encode(‘utf-8‘)).hexdigest()    return value@decoratordef get_datetime_from_unix(unix_time):    unix_time_value=unix_time    if not isinstance(unix_time_value,int):        unix_time_value=int(unix_time)    new_datetime=datetime.datetime.fromtimestamp(unix_time_value)    return new_datetime

以下是裝飾器decorators檔案的內容

# -*- coding: utf-8 -*-# @Time    : 2018/03/28 15:35# @Author  : cxa# @File    : decorators.py# @Software: PyCharmfrom functools import wrapsfrom logger.log import get_loggerimport tracebackdef decorator(func):    @wraps(func)    def log(*args, **kwargs):        try:            return func(*args, **kwargs)        except Exception as e:            get_logger().error("{} is error,here are details:{}".format(func.__name__,traceback.format_exc()))    return log

以下是headers檔案的內容

import randomfirst_num = random.randint(55, 62)third_num = random.randint(0, 3200)fourth_num = random.randint(0, 140)class FakeChromeUA:    os_type = [                ‘(Windows NT 6.1; WOW64)‘, ‘(Windows NT 10.0; WOW64)‘, ‘(X11; Linux x86_64)‘,                ‘(Macintosh; Intel Mac OS X 10_12_6)‘               ]    chrome_version = ‘Chrome/{}.0.{}.{}‘.format(first_num, third_num, fourth_num)    @classmethod    def get_ua(cls):        return ‘ ‘.join([‘Mozilla/5.0‘, random.choice(cls.os_type), ‘AppleWebKit/537.36‘,                         ‘(KHTML, like Gecko)‘, cls.chrome_version, ‘Safari/537.36‘]                        )headers = {    ‘User-Agent‘: FakeChromeUA.get_ua(),    ‘Accept-Encoding‘: ‘gzip, deflate, sdch‘,    ‘Accept-Language‘: ‘zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3‘,    ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8‘,    ‘Connection‘: ‘keep-alive‘}

以下是logger檔案的內容

# -*- coding: utf-8 -*-import osimport timeimport loggingimport syslog_dir1=os.path.join(os.path.dirname(os.path.dirname(__file__)),"logs")today = time.strftime(‘%Y%m%d‘, time.localtime(time.time()))full_path=os.path.join(log_dir1,today)if not os.path.exists(full_path):    os.makedirs(full_path)log_path=os.path.join(full_path,"t.log")def get_logger():     # 擷取logger執行個體，如果參數為空白則返回root logger     logger = logging.getLogger("t")     if not logger.handlers:            # 指定logger輸出格式            formatter = logging.Formatter(‘%(asctime)s %(levelname)-8s: %(message)s‘)            # 檔案日誌            file_handler = logging.FileHandler(log_path,encoding="utf8")            file_handler.setFormatter(formatter)  # 可以通過setFormatter指定輸出格式            # 控制台日誌            console_handler = logging.StreamHandler(sys.stdout)            console_handler.formatter = formatter  # 也可以直接給formatter賦值            # 為logger添加的Tlog器            logger.addHandler(file_handler)            logger.addHandler(console_handler)            # 指定日誌的最低輸出層級，預設為WARN層級            logger.setLevel(logging.INFO)     #  添加下面一句，在記錄日誌之後移除控制代碼     return  logger

一個python爬蟲工具類