Java reads multiple files in bulk and stores them in a database

Large amount of data in the file, want to put into storage, statistical analysis:

Code

Package Com.mobile.web.api;import Com.mobile.commons.jsonresp;import com.mobile.model.loginfo;import Com.mobile.service.loginfoservice;import Org.apache.commons.lang3.stringutils;import Org.apache.log4j.Logger; Import Org.springframework.beans.factory.annotation.autowired;import Org.springframework.transaction.annotation.transactional;import Org.springframework.web.bind.annotation.requestmapping;import Org.springframework.web.bind.annotation.requestmethod;import Org.springframework.web.bind.annotation.restcontroller;import java.io.*; import Java.text.parseexception;import java.text.simpledateformat;import Java.util.arraylist;import Java.util.date;import java.util.list;import Java.util.Locale; @RestController @requestmapping (Value="/test") @Transactional Public classImportcontroller {Logger log= Logger.getlogger ( This. GetClass ()); @AutowiredPrivateLoginfoservice Loginfoservice; @RequestMapping (Value="/importtxt", method =requestmethod.get) PublicJsonresp Importtxt () throws IOException, ParseException {log.debug ("Start importing Data"); String encoding="GBK"; List loginfolist=NewArrayList (); String dir="E:\\test\\log"; file[] Files=NewFile (dir). Listfiles ();  for(File file:files) {if(File.isfile () && file.exists ()) {//determine if a file existsimportfile (file, encoding, loginfolist); } Else {                returnJsonresp.tofail ("file does not exist, please check the file location! "); }} Boolean Insertflag=Loginfoservice.insertbatch (loginfolist); if(!Insertflag) {            returnJsonresp.tofail ("Save failed"); }        returnJsonresp.ok (); }
     Public Static voidimportfile (file file, String encoding, List loginfolist) throws IOException, ParseException {Inputstreamreade R Read=NULL;//considering the encoding format        Try{Read=NewInputStreamReader (NewFileInputStream (file), encoding); } Catch(unsupportedencodingexception e) {e.printstacktrace (); } Catch(FileNotFoundException e) {e.printstacktrace (); } BufferedReader BufferedReader=NewBufferedReader (read); String Linetxt=NULL; SimpleDateFormat SDF=NewSimpleDateFormat ("[Dd/mmm/yyyy:hh:mm:ss Z]", locale.us);  while((Linetxt = Bufferedreader.readline ())! =NULL) {string[] Linearr= Linetxt.split (" "); intLen =linearr.length; Loginfo Loginfo=NewLoginfo (); String logdate= linearr[0] +" "+ linearr[1]; System. out. println (Sdf.parse (logdate));//............. Time Conversion IssuesLoginfo.setlogtime (Sdf.parse (logdate)); Loginfo.setaccessip (linearr[2]); Loginfo.setproxyip (linearr[3]); Loginfo.setresponsetime (linearr[4]); Loginfo.setreferer (linearr[5].substring (1, linearr[5].length ()-1)); Loginfo.setmethod (linearr[6].substring (1)); Loginfo.setaccessurl (linearr[7].substring (0, linearr[7].length ()-1)); String Accessurl= linearr[7]; String[] Accessurlarr= Accessurl.split ("/"); Loginfo.setitemname (accessurlarr[3]); Loginfo.sethttpcode (linearr[8]); Loginfo.setrequestsize (linearr[9]); Loginfo.setresponsesize (linearr[Ten]); Loginfo.setcachehitstatus (linearr[ One]); String[] Uahead=NewString[len- -]; System.arraycopy (Linearr, A, Uahead,0, Len- -);            Loginfo.setuahead (Stringutils.join (Uahead)); Loginfo.setfiletype (Linearr[len-1]);        Loginfolist.add (Loginfo);    } read.close (); }}

File import, Success;

At this time, if the amount of data is particularly large, there will be slow storage, there is another way: After reading the file, the data in the desired format as a new file, and then use the SQL statement (or NAVICAT client) to import the file;

Java reads multiple files in bulk and stores them in a database