大資料拆分處理方式

標籤：

思路：
1：參考分頁方式將資料拆成指定大小線程數
2：在每個獨立的線程中去讀取資料並處理資料
步驟一實現

public class IndexIntiTools {        public static AtomicInteger runflag=new AtomicInteger();    //用於測試    public static List<Object>  syncList=new CopyOnWriteArrayList<Object>();    private static int idxThreadCount = 10;    private static Executor ex = Executors.newFixedThreadPool(idxThreadCount);    /**     * 構建索引     * @param hql     * @param size     */    public static void build(String hql, int size) {        int pagecount = idxThreadCount;        int count = size / pagecount;        int mod = size % pagecount;        List<Runnable> runList = new ArrayList<Runnable>(pagecount);        IndexExecutor idxExecutor;        for (int i = 0; i < pagecount; i++) {            if (i == (pagecount - 1)) {                idxExecutor = new IndexExecutor(hql, i * count, count + mod);            } else {                idxExecutor = new IndexExecutor(hql, i * count, count);            }            runList.add(idxExecutor);        }        for (Runnable runnable : runList) {            runflag.incrementAndGet();            ex.execute(runnable);        }    }}

步驟二實現

public class IndexExecutor implements Runnable{        private static final Log log = LogFactory.getLog(IndexExecutor.class);    private int start;    private int limit;    private String hql;    public IndexExecutor(String hql,int start, int limit) {        this.hql=hql;        this.start = start;        this.limit = limit;    }    @Override    public void run() {        log.info("hql:"+hql+",start:"+start+",limit"+limit);        //查詢資料庫(hql,start,limit);        log.info(list);        IndexIntiTools.syncList.addAll(list);        IndexIntiTools.runflag.decrementAndGet();    }}

大資料拆分處理方式