Database parallel Read and write (Python implementation)

Max_process = 16  #最大进程数  def read_sql_write (r_host,r_port,r_user,r_passwd,r_db,r_charset,w_ Host,w_port,w_user,w_passwd,w_db,w_charset,cmd,index=none):  #得到tem字典保存着信息  try:conn =  Pymysql. Connect (Host=r_host, Port=r_port, User=r_user, passwd =r_passwd, db =r_db, charset =r_charset) cursor  = conn.cursor () cursor.execute (cmd)  except Exception as e:error =  "[-] [-]%d fail to connect SQL for reading " % index log_error (' Error.log ', error)  return else:tem  = cursor.fetchone ()  print (' [+][+]%d succeed to connect SQL for reading '  % index)  finally:cursor.close () Conn.close ()  try:conn = pymysql. Connect (Host=w_host, Port=w_port, User=w_user, passwd =w_passwd, db =w_db, charset =w_charset) cursor  = conn.cursor () cursor.execute (cmd) except exception as e:error =  "[-][-]%d  fail to COnnect SQL for writing " % index Log_error (' Error.log ', error)  return else:print (' [+][+]%d  Succeed to connect SQL for writing '  % index) r_dict = dict () r_dict[' id '] = tem[0] r_ dict[' content_id '] = tem[1] r_dict[' pub_date '] = tem[2] r_dict[' title '] = cht_to_ CHS (Tem[3]) r_dict[' Title_score '] =tem[4]! [] (http://images2015.cnblogs.com/blog/1172464/201706/1172464-20170609000900309-1810357590.png) r_dict[' News_ Content '] = cht_to_chs (tem[5]) r_dict[' Content_score '] = tem[6] for key in  stock_dict.keys ():  #能找到对应的股票  if stock_dict[key][1] and  (r_dict[' title '].find ( STOCK_DICT[KEY][1])!=-1 or r_dict[' news_content '].find (stock_dict[key][1])!=-1 ): W_dict=dict () W _dict[' code '] = key w_dict[' english_name '] = stock_dict[key][0] w_dict[' cn_name '] =  stock_dict[key][1]  #得到分数  if r_dict[' Title_scorE ']: w_dict[' score ']=r_dict[' Title_score '] else:w_dict[' score ']=r_dict[' Content_score ']  #开始写入   Try: global max_process cmd =  "INSERT into Dyx_stock_score%d values ('%s ', '%s ',  %d  , '%s ', '%s ', %.2f ); "  % \ (index%max_process, r_dict[' content_id '], r_dict[' pub_date '], w_dict[' code ', w_dict[' english_name ') , w_dict[' Cn_name '], w_dict[' score ']) cursor.execute (cmd) conn.commit ()  except exception as e: error =  "[-]%d fail to write to SQL"  % index Cursor.rollback () log_error (' Error.log ', error)  else: print ("   [+]%d succeed to write to SQL"  % index) cursor.close () conn.close ()  def main ():  num = 238143# database queries get the total  p = none for index  in range (1,num+1):  if index%max_process==1: if p:p.close () p.join () p =  multiprocessing. Pool (max_process) r_cmd = (' Select Id,content_id,pub_date,title,title_score,news_content,content_score from dyx_emotion_analysis where id =  %d; '  %  (Index)) P.apply_async (func = read_sql_write,args= (r_host,r_port,r_user,r_passwd,r_db,r_ Charset,w_host,w_port,w_user,w_passwd,w_db,w_charset,r_cmd,index,))  if p:p.close () P.join ()

Database parallel Read and write (Python implementation)