Continuous sending of files between JAVA NIO servers (local beta)

Say in front: Give me the requirement is to implement a large number of files (approximately dozens of TB) from server A to send to Server B, the MD5 code in the a server to generate the file, and the MD5 authentication on Server B, verify by saving.

My idea of realization:

• Generate a TXT file for all file directories you want to upload, in the following format. Prefix, when the following number equals 9999, the preceding number is added by itself. (The purpose of this prefix is to be neat and to fail for easy querying.) ）

Aaa0000:d:\upload\addchannel.html

Aaa0001:d:\upload\addchannel2.html

Aaa0002:d:\upload\addcontactperson.html

Aaa0003:d:\upload\admin.html

Aaa0004:d:\upload\businessofchannel.html

....

Aaa9999:d:\upload\admin1.html

Aab0000:d:\upload\businessofchannel1.html

...

• Each time you read a directory, upload it.

• The part of the local beta that was not implemented did not write the successful and failed directories to the file, nor did they add logs.

The first part: Store the file directory in text and the folder is not stored.

import java.io.file;import java.io.fileoutputstream;public class readallpaths { Private static final string rootpath= "d:/upload/";  //the root path of  the files which will be copied private static final string  filepath= "G:/temp/unuploadedfilepath.txt";//the record of all files path/*  * the items of prefix and num construct the path  Prefix,for example aaa0001 * and it ' S mainly  convenient for  searching */private string prefix= "AAA";  private int num=0;/** *  main *  @param  args *  @throws  exception */public static void  main (String[] args)  throws Exception {ReadAllPaths  paths=new  Readallpaths (); File file=new file (fIlepath); if (File.exists ()) {File.delete ();} Fileoutputstream out=new fileoutputstream (file,true);p aths.getallpaths (rootPath, out); out.close ();} /** * get all path out *  @param  root *  @param  out  *  @throws  exception */private  void getallpaths (String root, Fileoutputstream out)  throws exception{file file=new file (root); if (File.isDirectory () {try{if (File.list (). length==0) {return;} Else{string[] files=file.list (); for (String f:files) {getallpaths (root+f+file.separator, out);}}} catch (NULLPOINTEREXCEPTION&NBSP;NPE) {return;}} Else{string pathnum=getpathnum (); String path=file.getabsolutepath (); Out.write ((pathnum+ ":" +path+ "\ n"). GetBytes ());}} /** * get the path prefix *  @return  */private String  Getpathnum () {stringbuilder sb=new stringbuilder (); Sb.append (Getprefix ()). Append (Getnum ()); SetNUm (); return sb.tostring ();} /** * get the string prefix of path prefix *  @return  */ Private string getprefix ()  {return prefix;} /** * set the string prefix of path prefix * for  Example:aaa aab aac .... Aaz aba .... Azz baa... */private void setprefix ()  {char[] ch=new char[3];ch=getprefix () . ToCharArray (); if (ch[2]!= ' Z ') {ch[2]++;} else{ch[2]= ' A '; if (ch[1]!= ' Z ') {ch[1]++;} else{ch[1]= ' A '; ch[0]++;}} Prefix=new string (CH);} /** * get the int prefix of path prefix *  @return  */ Private string getnum ()  {stringbuffer sb=new stringbuffer (); if (num<10) {sb.append (" "). append (num);} Else if (num<100) {sb.append ("xx"). append (num);} Else if (num<1000) {sb.append ("0"). Append (num);} Else{sb.append (num);} Return sb.tostring ();} /** * set the int prefix of path prefix * and the max  Num is 9999 and the min is 0000 */private void setnum ()  {if (num<9999) {num++;} Else{num=0;setprefix ();}}}

Part Two, server-side code

import java.io.ioexception;import java.net.inetsocketaddress;import  java.nio.channels.selectionkey;import java.nio.channels.selector;import  java.nio.channels.serversocketchannel;import java.util.iterator;public class server { selector selector = null; serversocketchannel serversocketchannel = null;private nioserverhandler2 handler; Public server ()  throws ioexception {selector = selector.open ();//  Open the server Socket Channel Serversocketchannel = serversocketchannel.open ();//  Adjust the blocking mode of the channel non-blocking serversocketchannel.configureblocking (false);//serversocketchannel.socket (). Setreuseaddress (True); Serversocketchannel.socket (). Bind (New inetsocketaddress (9999)); Serversocketchannel.register (selector,  selectionkey.op_accept);} Public server (Nioserverhandler2 handler)  throws ioexception {this (); This.handler  = handler;while  (Selector.select ()  > 0)  {iterator<selectionkey> it = selector.selectedkeys (). Iterator (); while   (It.hasnext ())  {selectionkey s = it.next (); It.remove (); This.handler.excute (( Serversocketchannel)  s.channel ());}} Public static void main (String[] args)  throws ioexception {new server ( New nioserverhandler2 ());}} public class nioserverhandler2 {private final static string directory  =  "g:\\niorequest\\";/** *  here we handle receive and send  *  *  @param   Serversocketchannel */public void excute (Serversocketchannel serversocketchannel)  { socketchannel socketchannel = null;try {socketchannel =  Serversocketchannel.accept (); //  waits for the client to connect Requestobject2 requestobject = receivedata ( Socketchannel);//  Data// logger.log (level.info,requestobject.tostring ()); String md5&nbSp;= digestutils.md5hex (Requestobject.getcontents ()); string response =  "";if  (Md5.equals (REQUESTOBJECT.GETMD5 ()))  {response =  ( New responseobject ("Succeed",  requestobject.getabsolutepath (),  "")). ToString (); File file = new file (Directory + requestobject.getrelativepath ());if  (! File.exists ())  {file.mkdirs ();} File file1 = new file (Directory + requestobject.getrelativepath ()  +  Requestobject.getfilename ());if  (!file1.exists ())  {file1.createnewfile (); Fileoutputstream fos = new fileoutputstream (File1); Fos.write (Requestobject.getcontents ()); Fos.close ();}  else {response =  (New responseobject ("Failed",  requestobject.getabsolutepath (),   "MD5 authentication failed"). toString ();} SYSTEM.OUT.PRINTLN (response); ResponseData (socketchannel, response);// logger.log (level.info,  Response);}  catch  (Ioexception e)  {e.printstacktrace ();}} /** * <p> *  reading the data in the channel to object  * </p> *  *  @param  socketChannel *  @return  *  @throws  ioexception */public requestobject2  receivedata (socketchannel socketchannel)  throws IOException {//  file name string  filename = null; string relativepath = null; string absolutepath = null; string md5 = null;//  file length int contentlength = 0;//  file contents byte[]  contents = null;//  because we parse the first 4 bytes is the file name length int capacity = 4; Bytebuffer buf = bytebuffer.allocate (capacity); int size = 0;byte[] bytes  = null;//  get the length of the filename size = socketchannel.read (buf);if  (size >= 0)  {buf.flip (); Capacity = buf.getint (); Buf.clear ();}   Take the file name, believe that the file name can be read once, if you have a file name more than 1k  you are sick buf = bytebuffer.allocate (capacity); Size = socketchannel.read (BUF);if  (size >=  0)  {buf.flip (); bytes = new byte[size];buf.get (bytes); Buf.clear ();} String fileinfo = new string (bytes); System.out.println (FileInfo); Filename = fileinfo.split (";") [0];relativepath = fileinfo.split (";") [1];absolutepath = fileinfo.split (";") [2];md5 = fileinfo.split (";") [3];//  Get file length capacity = 4;buf = bytebuffer.allocate (capacity);size =  Socketchannel.read (BUF);if  (size >= 0)  {buf.flip ();//  file length is not possible, If you want to do calibration you can leave Capacity = buf.getint (); Buf.clear ();} if  (capacity == 0)  {contents = new byte[] {};}  else {//  to receive a byte array in buffer bytearrayoutputstream baos = new  The Bytearrayoutputstream ();//  file may be very large// capacity = 1024;buf = bytebuffer.alloCate (capacity);while  ((Size = socketchannel.read (BUF))  >= 0)  {buf.flip (); Bytes = new byte[size];buf.get (bytes); baos.write (bytes); Buf.clear ();} Contents = baos.tobytearray ();} Requestobject2 requestobject = new requestobject2 (filename, relativepath,  absolutepath, md5, contents); return requestobject;} Private void responsedata (Socketchannel socketchannel, string response)  { Bytebuffer buffer = bytebuffer.wrap (Response.getbytes ()); try {socketchannel.write (buffer); Buffer.clear ();//  confirm that the thing to send is finished sending off the output  otherwise it will receive Socketchannel.read (Buffer)//  is likely to cause blocking   You can comment out this (L) and find that the client has been waiting to receive data Socketchannel.socket (). Shutdownoutput ();//  (L)} catch  (IOException  e)  {e.printstacktrace ();}}} import java.io.serializable;public class requestobject2 implements serializable  {private static fInal long serialversionuid = 1l;private string filename;private string  relativepath;private string absolutepath;private string md5;private byte[]  contents;public requestobject2 (string filename, string relativepath, string  absolutepath, string md5, byte[] contents)  {this.filename = filename; This.relativepath = relativepath;this.absolutepath = absolutepath;this.md5 = md5 ; this.contents = contents;} Public string getfilename ()  {return filename;} Public string getrelativepath ()  {return relativepath;} Public string getabsolutepath ()  {return absolutepath;} PUBLIC&NBSP;STRING&NBSP;GETMD5 ()  {return md5;} Public byte[] getcontents ()  {return contents;}

Part Three client code

import java.io.bufferedreader;import java.io.file;import java.io.filereader;import  Java.io.ioexception;import java.net.inetsocketaddress;import java.nio.channels.selectionkey;import  java.nio.channels.selector;import java.nio.channels.socketchannel;public class client2  {private static final String unpath =  "G:\\temp\\unuploadedfilepath.txt"; private static final string pathpre =  "d:\\upload\\";p rivate static  final string ipaddr =  "127.0.0.1";p rivate static final int port =  9999; Selector selector;public client2 ()  throws IOException {selector =  Selector.open (); New thread (New senddatarunnable ()). Start (); private class senddatarunnable implements runnable {private clienthandler  Handler;public senddatarunnable ()  {handler =  New clienthandler ();} @Overridepublic  void run ()  {try {BufferedReader reader = new  BufferedReader (New filereader (New file (Unpath))); string path =  "";while  ((Path = reader.readline ())  != null & & path.length ()  != 0)  {SocketChannel socketChannel;socketChannel =  Socketchannel.open (); Socketchannel.connect (New inetsocketaddress (Ipaddr, port)); Socketchannel.configureblocking (false); Socketchannel.register (Selector, selectionkey.op_read); Handler.senddata (Socketchannel, path, pathpre); String response = handler.receivedata (Socketchannel); SYSTEM.OUT.PRINTLN (response); Socketchannel.close ();}}  catch  (exception e)  {e.printstacktrace ();}} Public static void main (String[] args)  throws IOException {Client2  Client = new client2 ();}} Import java.io. bytearrayoutputstream;import java.io.file;import java.io.fileinputstream;import  java.io.ioexception;import java.nio.bytebuffer;import java.nio.channels.socketchannel;import  org.apache.commons.codec.digest.digestutils;public class clienthandler {public void  SendData (Socketchannel socketchannel,string path,string pathpre) throws Exception{ SYSTEM.OUT.PRINTLN (path); String absolutefilepath=getabsolutefilepath (path); String filename=getfilename (Absolutefilepath); String relativefilepath=getrelativefilepath (Absolutefilepath, pathpre,filename); System.out.println (Absolutefilepath); byte[] bytes=makefiletobytes (Absolutefilepath); System.out.println (bytes.length); String md5=digestutils.md5hex (bytes); String fileinfo=new stringbuffer (). Append (FileName). Append (";"). Append (RelativeFilePath). Append (";"). Append (Path). Append (";"). Append (MD5). toString (); System.out.println (FileInfo); Bytebuffer&nbSp;buffer = bytebuffer.allocate (8 +fileinfo.getbytes (). length+bytes.length); Buffer.putInt ( Fileinfo.getbytes (). length); Buffer.put (Fileinfo.getbytes ()); Buffer.putint (bytes.length); Buffer.put ( Bytebuffer.wrap (bytes)); Buffer.flip (); socketchannel.write (buffer); Buffer.clear ();//  off the output stream to prevent blocking on acceptance, is to tell the receiver that the content has been sent out, you do not have to wait socketchannel.socket (). Shutdownoutput (); Private string getabsolutefilepath (String path) {return path.substring (8);} Private string getrelativefilepath (string absolutefilepath,string pathpre,string  FileName) {return absolutefilepath.substring (Pathpre.length (), Absolutefilepath.length ()-fileName.length ()) ;} Private string getfilename (String path) {return new file (path). GetName ();} Private byte[] makefiletobytes (String filepath) {file file=new file (FilePath); byte[]  ret = null;          try {  & nbsp;             fileinputstream in  = new fileinputstream (file);               bytearrayoutputstream out = new bytearrayoutputstream (4096);               byte[] b = new byte[4096 ];              int n;               while  ((N = in.read (b))  != -1)  {                   out.write (b, 0, n);               }               in.close ();              out.close ();               ret = out.tobytearray ();           } catch  (ioexception e)  {               // log.error ("helper: Get bytes from file process error! ");               e.printstacktrace ();           }           return ret;  }public string receivedata (Socketchannel socketchannel)   Throws ioexception {bytearrayoutputstream baos = new bytearrayoutputstream (); string response =  ""; try {bytebuffer buffer  = bytebuffer.allocate (1024x768);byte[] bytes;int count = 0;while  (count =  socketchannel.read (buffer))  >= 0)  {buffer.flip (); bytes = new byte[count ];buffer.get (bytes); baos.write (bytes); Buffer.clear ();} Bytes = baos.tobytearray (); Response = new string (bytes,  "UTF-8");// Socketchannel.socket (). Shutdowninput ();}  finally {try {baos.close ();}  catch  (Exception ex)  {}}return response;}}

/* This is all done, the comments are not enough, and some of the code is from the Internet. Later time will be full comment, or the next time directly on the final use of the code * *

This article is from the "Maple Leaf is not red" blog, please be sure to keep this source http://itlearninger.blog.51cto.com/12572641/1945045

Continuous sending of files between JAVA NIO servers (local beta)