Taking multiple processes to read OSS compliant data as an example, the use of multi-process communication, the acquisition of multi-process data

Import datetimeimport sysimport oss2from itertools import Isliceimport pandas as Pdimport reimport jsonfrom pandas.tseries  . Offsets import dayfrom multiprocessing import Process, Joinablequeue, Cpu_count, Managerimport timedef Mkbuck (BK): auth = Oss2. Auth (username, password) bucket = Oss2. Bucket (auth, address, BK) return bucket# get the last hour of the day before the Pathsdef getbflastpt (bucket, bfyespattern): Bfpamax = []for BF in Islice (Oss2. Objectiterator (bucket, Prefix=bfyespattern), sys.maxsize): c = bf.keyif c[-1:]! = '/': bfpamax.append (int (c.split ('/') [ 4]) last = PD.  Series (Bfpamax). Unique (). Max () if last < 10:BFLASTPT = Bfyespattern + '/0 ' + str (last) else:bflastpt = Bfyespattern + '/' + STR (last) return bflastpt# gets the day of the first one hours of Pathsdef getnowfirstpt (bucket, nowpattern): Bfpamin = []for bf in Islice (Oss2. Objectiterator (bucket, Prefix=nowpattern), sys.maxsize): c = bf.keyif c[-1:]! = '/': bfpamin.append (int (c.split ('/') [4] ) First = PD. Series (Bfpamin). Unique () min () if first < 10:NOWFIRSTPT = Nowpattern + '/0 ' + str (first) else:nowfirstpt = Nowpattern + '/' + str (FIRST) return nowfirstpt# get all the paths of yesterday and merge to get complete paths and number Def getfullnum (BK, Bfyespattern, Nowpattern, yespattern): Lists = []bucket = Mkbuck (BK) Bfyespattern = GETBFLASTPT (bucket, Bfyespattern) Nowpattern = GETNOWFIRSTPT (bucket, nowpattern) Timelist = (s for s in (Bfyespattern, Yespattern, Nowpattern)) for Pter in Ti Melist:for BF in Islice (Oss2. Objectiterator (bucket, prefix=pter), sys.maxsize): c = Bf.keylists.append (c) return lists, Len (lists) #以下为进程间通信, i.e. producer, Consumer Model def getfull (BK, Bfyespattern, Nowpattern, Yespattern, Q): lists, num = Getfullnum (BK, Bfyespattern, Nowpattern, Yespa Ttern) for C in Lists:q.put (c) q.join () def consumer (BK, Q, d): Bucket = Mkbuck (BK) repattern2 = Re.compile (' {. * ' Adadji ',. *} ' While true:js = []ress = Q.get () if ress[-1:]! = '/': Remote_data = Bucket.get_object (ress). Read (). Decode (' utf-8 ') AA = (d f or d in Repattern2.findall (Remote_data)) is a in aa:temdic = Json.loads (a) if (StartTime <= temdic[' Created_at ')) and (t emdic[' Created_at '] <= endtime): Js.append (temdic) df = PD. DataFrame (JS, columns=[' DD ', ' CC ']) d[ress] = df# #d为通过主进程Manager共享变量将数据取出 # print (ress) q.task_done () # sends a signal to Q.join (),  Prove that a data has been taken away if __name__ = = ' __main__ ': S1 = time.time () Now_time = Datetime.datetime.now () # Gets the current time Bfyes_time = (Now_time- 2 * Day ()). Strftime ('%y/%m/%d ') Yes_time = (now_time-1 * Day ()). Strftime ('%y/%m/%d ') yesdate = (now_time-1 * Day ()). Str Ftime ('%y-%m-%d ') yesdate1 = (now_time-1 * Day ()). Strftime ('%y%m%d ') Endtime = (now_time-1 * Day ()). Strftime ('%y-%m-%d 2 3:59:59 ') StartTime = (now_time-1 * Day ()). Strftime ('%y-%m-%d 00:00:00 ') nowdate = Now_time.strftime ('%y/%m/%d ') bk = ' xxx ' Bfyespattern = '%s/%s '% (BK, bfyes_time) Yespattern = '%s/%s '% (BK, yes_time) Nowpattern = '%s/%s '% (BK, nowdate) q = Joi Nablequeue (Cpu_count ()) m = Manager () d = m.dict () # #主进程字典共享p1 = Process (Target=getfull, args= (' xx ', Bfyespattern, Nowpatt Ern, Yespattern, Q) # # # # # # # # #生成consumer多进程cc = []for C in range (Cpu_count ()-1): C1 = Process (Target=consumer, args= (' xx', Q, D)) Cc.append (c1) p_l = [P1]for c in Cc:c.daemon = Truep_l.append (c) for P in P_l:p.start () p1.join () d = d.values () df1 = Pd.concat (d, Ignore_index=true) df1.sort_values (' Created_at ', inplace=true) print (Time.time ()-S1) print (' = ' * 20) Print (DF1)

Description: Demand for yesterday's data can be, because the OSS real-time data storage may exist in advance or delay, so read the last hour of the day before yesterday all, the day the first hour of data, the reader can be modified according to their own circumstances

Taking multiple processes to read OSS compliant data as an example, the use of multi-process communication, the acquisition of multi-process data