#!/usr/bin/env python2.7#-*- encoding: utf-8 -*-import timeimport datetimeimport MySQLdbimport osimport reimport paramiko# map的value值增量def safe_inc_map_val(map, key, val): if key in map.keys(): map[key] = map[key] + val else: map[key] = val# 目前時間def time_now(): return datetime.datetime.now()# 判斷檔案是否存在def check_file_exists(filename): if not os.path.exists(filename): return False return True # 刪除檔案def remove_file(file_name): if(os.path.isfile(file_name)): os.remove(file_name)# 刪除相應目錄下的txt檔案def remove_text_file(filepath): filelist = os.listdir(filepath) for ele in filelist: if ele.find('.txt')>-1: os.remove(filepath+ele) #IP轉化為對應整數def ip_to_int(ip): int_val = reduce(lambda x,y:(x<<8)+y,map(int,ip.split('.'))) return int_val#IP轉化為對應整數def Ip2Int(ip): import struct,socket return struct.unpack("!I",socket.inet_aton(ip))[0]# 計算文字檔的行數def line_count(file_name): count = -1 #讓空檔案的行號顯示0 for count,line in enumerate(open(file_name)): pass #enumerate格式化成了元組,count就是行號,因為從0開始要+1 return count+1# 使用外部系統程式 wc -l, 來計算文字檔的行數def linecount_wc(file_name): cmd = 'wc -l %s'%file_name return int(os.popen(cmd).read().split()[0])# 遠程拷貝檔案def sftp_get(host_ip,remote_path,local_path,username,password): t = paramiko.Transport((host_ip,22)) t.connect(username=username, password=password) sftp = paramiko.SFTPClient.from_transport(t) src = remote_path des = local_path sftp.get(src,des) t.close()# 按行分割文字檔def split_file(filepath, file_num): total_rows = linecount_wc(filepath) lines = total_rows/file_num+1 #這裡大小加上1row,保證最後一個分割的檔案不會丟資料 command = " split -l%d -a2 -d %s %s" % (lines, filepath, filepath) #其實使用的就是liunx的split #print command os.system(command)# 按行分割文字檔,並將分割後的檔案輸出到新目錄def split_file2(filepath, new_filepath, file_num): total_rows = linecount_wc(filepath) lines = total_rows/file_num+1 command = " split -l%d -a2 -d %s %s" % (lines, filepath, new_filepath) os.system(command)# 返回結果集中的int值def query_fetch_one_int(cursor, sql): result = 0 n = cursor.execute(sql) if (n > 0): r = cursor.fetchone() if r[0] != None:result = int(r[0]) return result# 返回結果集中的float值def query_fetch_one_float(cursor, sql): result = 0 n = cursor.execute(sql) if (n > 0): r = cursor.fetchone() if r[0] != None:result = float(r[0]) return result# 返回結果集中的kv值列表def query_fetch_pair_array(cursor, sql): result = [] n = cursor.execute(sql) if (n > 0): ds = cursor.fetchall() for r in ds:result.append((r[0], r[1])) return result# 返回結果集中的一組列表def query_fetch_one_list(cursor , sql): result = [] n = cursor.execute(sql) if n>0: ds = cursor.fetchall() result = [ele[0] for ele in ds if ele!=None] return result