java csv list cant not repeat

標籤：delete   blog   div   ant   extern   exists   except   nta   path   

require:

/** * before: * file A1.csv {1,2,3,4,5} * file A2.csv {2,3,9,10,11} * file B1.csv {5,12,13,14,15} * file B2.csv {16,14,15,4,9,20,30} * A1.csv A2.csv A3.csv A4.csv cant not repeat * * after: * file A1.csv {1,4} * file A2.csv {2,3,10,11} * file B1.csv {12,13} * file B2.csv {16,9,20,30} */

　　

[email protected]:~/csvrepeat$ lsA1.csv  A2.csv  B1.csv  B2.csv

　　

 

CSVUtilVersion2.java

import lombok.extern.slf4j.Slf4j;import org.apache.commons.collections.CollectionUtils;import org.apache.commons.io.FileUtils;import org.apache.commons.io.LineIterator;import org.springframework.util.StringUtils;import java.io.*;import java.lang.reflect.Array;import java.util.*;/** * before: * file A1.csv {1,2,3,4,5} * file A2.csv {2,3,9,10,11} * file B1.csv {5,12,13,14,15} * file B2.csv {16,14,15,4,9,20,30} * A1.csv A2.csv A3.csv A4.csv cant not repeat * * after: * file A1.csv {1,4} * file A2.csv {2,3,10,11} * file B1.csv {12,13} * file B2.csv {16,9,20,30} */@Slf4jpublic class CSVUtilVersion2 {    private static final String CSV_PATH = "/home/tangxin/csvrepeat/";    private static final boolean CREATE_SWITCH = true;    /**     * read single column data list     * @param path     * @return     */    public static List<String> ids(String path) {        List<String> result = new ArrayList<>();        File csv = new File(path);  // CSV檔案路徑        LineIterator it = null;        try {            it = FileUtils.lineIterator(csv);            while (it.hasNext()) {                String line = it.nextLine();                if (line.trim().contains("ID")) {                    continue;                }                String[] arr = line.split(",");                String ID = arr[0];                ID = ID.replaceAll("\"", "").trim();                if (!StringUtils.isEmpty(ID)) {                    result.add(ID);                }            }        } catch (Exception e) {            log.error("讀取ID csv檔案失敗:{}", e.getMessage());        } finally {            LineIterator.closeQuietly(it);        }        return result;    }    /**     * from src delete oth     * @param src     * @param oth     * @return     */    public static List removeAll(List src, List oth) {        LinkedList result = new LinkedList(src);        HashSet othHash = new HashSet(oth);        Iterator iter = result.iterator();        while (iter.hasNext()) {            if (othHash.contains(iter.next())) {                iter.remove();            }        }        return result;    }    /**     * -Xms1g -Xmx1g -XX:PermSize=128m -XX:SurvivorRatio=2 -XX:+UseParallelGC     * @param args     * @throws Exception     */    public static void main(String[] args) throws Exception {        //∑=1+2+3+...+(n-1) group        LinkedList<String> fileList = new LinkedList<>();        fileList.add("A1.csv");        fileList.add("A2.csv");        fileList.add("B1.csv");        fileList.add("B2.csv");//        fileList.add("C1.csv");        DescartesRepeat(fileList);        ded(fileList);    }    private static void DescartesRepeat(LinkedList<String> fileList) {        Set<String> repeatList = new HashSet<>();        Set<String> groupSet = new HashSet<>();        Set<String> goONList = new HashSet<>();        //A1->A2,B1,B2        for (int i = 0; i < fileList.size(); i++) {            String itemI = fileList.get(i);            for (int j = 0; j < fileList.size(); j++) {                String itemJ = fileList.get(j);                if (!itemI.equals(itemJ)) {                    String groupR1 = itemI + "->" + itemJ;                    String groupR2 = itemJ + "->" + itemI;                    if (groupSet.contains(groupR1) || groupSet.contains(groupR2)){                        continue;                    }                    groupSet.add(groupR1);                    String repeatT = repeat(CSV_PATH + itemI, CSV_PATH + itemJ);                    if(!StringUtils.isEmpty(repeatT)){                        repeatList.add(repeatT);                        //System.out.println(groupR1+"->"+repeatT);                    }                }            }        }        if (CollectionUtils.isNotEmpty(repeatList)) {//            System.out.println(repeatList);            for (String repeatItem : repeatList) {                Iterator<String> iterator = fileList.iterator();                while (iterator.hasNext()) {                    String oldItem = iterator.next();                    String oldS = oldItem.replace(".csv", "").replace("-new","");                    String repeatS = repeatItem.replace(".csv","").replace("-new","");                    if (repeatS.contains(oldS)) {                        iterator.remove();                        goONList.add(repeatItem);                    }                }            }            fileList.addAll(goONList);            System.out.println(fileList);            DescartesRepeat(fileList);        }    }    public static void ded(List<String> args) {        //保證指定csv列表每組都不能有重複資料        for (int i = 0; i < args.size(); i++) {//            if(i>0){//                continue;//            }            String source = CSV_PATH + args.get(i);            for (int j = 0; j < args.size(); j++) {                if (i == j) {                    continue;                }                String target = CSV_PATH + args.get(j);                intersection(source, target);            }        }    }    public static void intersection(String sourcePath, String targetPath) {        List<String> ids1 = ids(sourcePath);        List<String> ids2 = ids(targetPath);        List<String> inter = (List<String>) CollectionUtils.intersection(ids1, ids2);        System.out.println(sourcePath + "和" + targetPath + "的重複資料大小" + inter.size());    }    public static String repeat(String source, String target){        //cdd fund xyd        List<String> ids1 = ids(source);        List<String> ids2 = ids(target);//        System.out.println(source + "集合大小" + ids1.size());//        System.out.println(target + "集合大小" + ids2.size());        List<String> inter = (List<String>) CollectionUtils.intersection(ids1, ids2);//        System.out.println("去重資料大小:" + inter.size());        if (inter != null && inter.size() > 0) {            if (ids1.size() > ids2.size()) {                return repeatInner(source, ids1, inter);            } else if (ids2.size() > ids1.size()) {                return repeatInner(target, ids2, inter);            } else {                return repeatInner(source, ids1, inter);            }        }        return "";    }    private static String repeatInner(String source, List<String> ids, List<String> inter) {        String newPath = source.replace(".csv", "-new.csv");        List<String> ids1new = removeAll(ids, inter);        createCSV(ids1new, newPath);        return newPath.replace(CSV_PATH,"");    }    /**     * 建立CSV檔案     */    public static void createCSV(List<String> list, String fileName) {        if(!CREATE_SWITCH){//            System.out.println("建立csv開關關閉");            return;        }else{//            System.out.println("建立csv開關開啟");        }        // 表格頭        Object[] head = {"ID"};        List<Object> headList = Arrays.asList(head);        //資料        List<List<Object>> dataList = new ArrayList<>();        List<Object> rowList = null;        for (int i = 0; i < list.size(); i++) {            rowList = new ArrayList<>();            rowList.add(list.get(i));            dataList.add(rowList);        }        File csvFile;        BufferedWriter csvWtriter = null;        try {            csvFile = new File(fileName);            File parent = csvFile.getParentFile();            if (parent != null && !parent.exists()) {                parent.mkdirs();            }            csvFile.createNewFile();            // GB2312使正確讀取分隔字元","            csvWtriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), "GB2312"), 1024);            // 寫入檔案頭部            writeRow(headList, csvWtriter);            // 寫入檔案內容            for (List<Object> row : dataList) {                writeRow(row, csvWtriter);            }            csvWtriter.flush();        } catch (Exception e) {            e.printStackTrace();        } finally {            try {                csvWtriter.close();            } catch (IOException e) {                e.printStackTrace();            }        }    }    /**     * 寫一行資料     *     * @param row       資料列表     * @param csvWriter     * @throws IOException     */    private static void writeRow(List<Object> row, BufferedWriter csvWriter) throws IOException {        for (Object data : row) {            StringBuffer sb = new StringBuffer();            String rowStr = sb.append("\"").append(data).append("\",").toString();            csvWriter.write(rowStr);        }        csvWriter.newLine();    }}

　　

[email protected]:~/csvrepeat$ lsA1.csv  A1-new.csv  A1-new-new.csv  A2.csv  A2-new.csv  B1.csv  B2.csv  B2-new.csv  B2-new-new.csv

　　

 

java csv list cant not repeat