Big Data split processing mode

Ideas:
1: Refer to Paging to split data into a specified size number of threads
2: Read the data and process the data in each separate thread
Step One implementation

 Public classIndexintitools { Public StaticAtomicinteger runflag=NewAtomicinteger (); //for testing     Public StaticList<object> synclist=NewCopyonwritearraylist<object>(); Private Static intIdxthreadcount = 10; Private StaticExecutor ex =Executors.newfixedthreadpool (Idxthreadcount); /*** Build Index *@paramHQL *@paramsize*/     Public Static voidBuild (String hql,intsize) {        intPageCount =Idxthreadcount; intCount = Size/PageCount; intMoD = size%PageCount; List<Runnable> runlist =NewArraylist<runnable>(PageCount);        Indexexecutor Idxexecutor;  for(inti = 0; i < PageCount; i++) {            if(i = = (pagecount-1) ) {Idxexecutor=NewIndexexecutor (HQL, I * count, Count +MoD); } Else{idxexecutor=NewIndexexecutor (HQL, I *count, Count);        } runlist.add (Idxexecutor); }         for(Runnable runnable:runlist) {runflag.incrementandget ();        Ex.execute (runnable); }    }}

Step Two implementation

 Public classIndexexecutorImplementsrunnable{Private Static FinalLog log = Logfactory.getlog (indexexecutor.class); Private intstart; Private intlimit; PrivateString hql;  PublicIndexexecutor (String hql,intStartintlimit) {         This. hql=hql;  This. Start =start;  This. Limit =limit; } @Override Public voidrun () {Log.info ("HQL:" +hql+ ", Start:" +start+ ", limit" +limit); //Query Database (hql,start,limit);log.info (list);        IndexIntiTools.syncList.addAll (list);    IndexIntiTools.runflag.decrementAndGet (); }}

Big Data split processing mode