Personal first open source distributed project Distributetemplate implementation of three network communication Netty transfer large files

Today I will talk about network communication, here my initial version due to the things Netty framework so here is how the network Netty in our here is how to use, next week to start adding RPC Lucene content after the implementation of the 0.2 0.3 version, the following will be removed Netty dependent on the use of native NIO2 (AIO) asynchronous non-blocking way to implement their own network communication, that is, this part may be implemented a simple but netty thin and efficient network framework, late to make a separate branch may open open source, Netty plainly is the event-driven and NIO plus some protocol and exception handling, nonsense does not Said more.

I recently had an idea to add a distributed lock feature I'm recently trying to implement a distributed lock, because recently RPC is trying to implement the following distributed lock Test, of course, the primary level of the lock is implemented first, the whole queue after that is RABBITMQ those message middleware will also try to Implementation of

The first 2 days I added a configuration class, in order to better implement user management of the configuration

Public  class configbuilder {private static configbuilder configbuilder=new  configbuilder ();//Host List private list<inetsocketaddress> address=lists.newarraylist (); private inetsocketaddress currenthost=null;private list<serviceconfig> services= Lists.newarraylist ();//alias Protected bimap<string,inetsocketaddress> alias=hashbimap.create (); Private string configfile;    public configbuilder ()  {     configfile=context.defaultconfig;    config config=configfactory.load ( Context.defaultconfig);     initconfig (config);     }private void  initconfig (Config config)  {list<? extends configobject> nodes= Config.getobjectlist ("Server.hosts");    for  (configobject node :  nodes) &NBSP;{&NBSP;&NBSP;&NBSP;&NBSP;&NBSP;&NBsp;integer remoteport=integer.parseint (Node.get ("RemotePort"). Render ());          string remoteip=node.get ("RemoteHost"). Unwrapped (). toString ();//Remote host ip          string name=node.containskey ("name")? Node.get ("Name"). Unwrapped (). toString (): remoteip;//host alias          inetsocketaddress  host=new inetsocketaddress (Remoteip, remoteport);          address.add (host);           alias.put (name,  host);          //todo  Get            for (servicetype servicetype :  ServiceConfig.ServiceType.values ()) {           string  servicename=servicetype.name (). toLowerCase ();  &nBsp;        if (Node.containskey (serviceName)) {           hashmap fcs= (HASHMAP)  node.get (serviceName). Unwrapped ();           serviceconfig serviceconfig=new serviceconfig ( );           serviceconfig.setservicetype (ServiceType);           serviceconfig.setinfo (FCS);           services.add (ServiceConfig);           }          }              }    string chost=config.getstring (" Client.currenthost "), Int port=config.getint (" Client.currentport "); Currenthost=new inetsocketaddress ( Chost, port);} PubLic configbuilder (String configfile)  {this.configFile=configFile; Config config=configfactory.load (configfile); Initconfig (config);} Public static configbuilder getinstance () {if (configbuilder==null) {return new  Configbuilder ();} Else{return configbuilder;}} Public static configbuilder getinstance (String configfile) {if (configBuilder==null) {return  new configbuilder (configfile);} Else{return configbuilder;}} /** *  Add Server host  *  @param  host *  @param  port *  @return  *   Add (Modify) Person: Zhuyuping */public configbuilder addhost (string host,int port) { Inetsocketaddress h=new inetsocketaddress (Host, port); Address.add (h); Alias.put (host, h); Return this;} /** *  add host and set   host alias  *  @param  host *  @param  port *  @param  nname *  @return  *  Add (Modify) Person: zhuyuping */    public configbuilder addhost (string host,int port,string  Nname) {      addhost (host, port);       Alias.put (Nname, new inetsocketaddress (Host, port)); return this;}     /**     *  Remove Host      * @ param host     *  @param  port     * @ return     *  Add (Modify) person:zhuyuping     */     public configbuilder removehost (String host,int port) {     Inetsocketaddress h=new inetsocketaddress (Host, port); Alias.inverse (). Remove (h); Address.remove ( h); return this;}     /**     *  set the current host name   and Port        *  @param  host     *  @param  port     *  @return       *  Add (Modify) person:zhuyuping     */    public  Configbuilder setself (String host,int port) {currenthost=new inetsocketaddress (host,  port); return this;}     /**     *  writing to the configuration file      *       *  Add (Modify) person:zhuyuping     */     Public void buildtofile () {        string path= Thread.CurrentThread (). Getcontextclassloader (). GetResource ("") +configfile;    try { Writer out = new bufferedwriter (New outputstreamwriter (New FileOutputStream (path ) (,  "UTF8")); Stringbuilder sb=new stringbuilder ("server{"), Sb.append ("\n\t"). Append ("hosts=["); int&nbSp;i=0;int size=address.size ();for  (inetsocketaddress host : address)  {sb.append ("\t {"); Sb.append ("Name="). Append (Alias.inverse (). Get (host)). Append ("\n\t"); Sb.append ("remotehost="). Append (Host.gethoststring ()). Append ("\n\t"), Sb.append ("remoteport="). Append (Host.getport ()). Append ("\n\t");  sb.append ("\T&NBSP;}");     if (i!=size) {    sb.append (",");     }}sb.append ("]"). Append ("\n\t"). Append ("}");   //continue   Save client      sb.append ("\n\t"). Append ("client{"). Append ("\n\t"). Append ("currenthost="). Append ( Currenthost.gethoststring ()). Append ("\n\t"),    sb.append ("\n\t"). Append ("currentport="). Append ( Currenthost.getport ()). Append ("\n\t");    sb.append ("}");    out.write (sb.toString ());}  catch  (exception e)  {}             }   &nbsP;    public iconfig bulid (Context context) {             return null;    }    @ Overridepublic string tostring ()  {return  "configbuilder [address="  +  address +  ",  currenthost=" + currenthost +  ",  services="  +  services +  ",  alias="  + alias+  ",  configfile="  + configFile  +  "]";} Public static void main (String[] args)  {configbuilder build= Configbuilder.getinstance (); Build.addhost ("192.168.8.8",  1234). Addhost ("192.168.9.9", 2222,  " Test "); SYSTEM.OUT.PRINTLN (build);}}

In this way the user is easy to set up the configuration file, this weekend I will add such a way to the distributetemplate, so that you can use the query object can also be passed SQL for the corresponding additions and deletions to change and synchronization.

OK, let's see how our network communication is going to come true.

First of all, the configuration context is good, we think that network communication is not the core throughout the use of, since then we do not make service services

So I've defined a service

 /** *  *       *      *   @author  zhuyuping        *  @version  1.0       *  @created  2014-7-9  afternoon 12:56:04  *  @function: Put   Some   commonly used network communication as the basis   through other customer service   drive   for basic access   function    package into service   to   services to resolve    */public interface service {    /**     *   Start Service      *      *  Add (Modify) person:zhuyuping      */void start ();/** *  Close service   *  *  Add (Modify) Person: zhuyuping */void stop (); /** *  service name  *  @return  *  Add (Modify) Person: Zhuyuping */string getname (); Void setname (string name);/** *  get configuration    *  @return  *  Add (Modify) Person: ZhuyupinG */iconfig getconfig ();/** *  injection configuration  */void setconfig (IConfig config);     /**     *  go to host   Add File      * @ param address     *  @param  build     *  Add (Modify) Person: Zhuyuping     */void insert (list<inetsocketaddress> address, Immutablemap<string, object> build);     /**     *   Delete files   or folders      *  @param  address     *   @param  build     *  Add (Modify) Person: zhuyuping     */ Void delete (List<inetsocketaddress> address,immutablemap<string, object> build );    /**     *  sync files or folders   If the user does not have delivery criteria   Then the default is the sync area      *  @param  address     *  @param  build      *  Add (Modify) Person: Zhuyuping     */void sync (list<inetsocketaddress > address,immutablemap<string, object> build);}

And then this service to us to use the time to inject into it, so that is I want the network network services to come, I want to girlfriend and then girlfriend did not come I want to XXX service can be obtained, and this service is only initialized in the initial frame, such a situation, we think of a pattern

Think of it.

Flyweight mode

That's it, because the service is dependent on the node, so I'm using the node, and I want to get out of the way.

Public IConfig GetConfig () {return config;}    public void Setconfig (IConfig config) {this.config = config;} Public nodefactory (iconfig config) {super (); this.config = config;} Private IConfig config;private static map<string,node> caches=maps.newhashmap ();p ublic Node CreateNode (Class <? Extends Node> clazz) {Node nd=null;if (Clazz.isassignablefrom (Filesystem.class)) {nd=caches.get (filesystem.name); if (nd==null) {nd=new FileSystem (config); Caches.put (Filesystem.name, ND);}} .... return nd;}}

In other words, the use of map cache to avoid creating too many objects wasted memory, this situation is used in many occasions, especially the connection pool

Let's take a look at how to achieve communication, event-driven supervisor hears the time after the queue rotation and then hand over to the service processing

Then this part is given to the service Management service to provide access only to send data.

So the service inside needs some access to communication needs if you use TCP BIO just need to simply create a socket and serversocket can be used to send the object file below, about Netty How to create basic use can refer to my blog spring and Netty simple integration, where there is a problem is eventloop not separated to know friends, remember separate, I would not bother to re-update,

Let's see how the methods are sent in the service.

Here check whether the connection is connected, and then send no connection sent to the dead letter queue, because the dead letter queue is a weak reference and then to the unreal reference, and finally back to the Tao Unreal reference queue, finally persisted record, this is similar to the MySQL log mode, if the user if the reference can traverse to go if the reference to find, Can not be traversed to the Unreal reference queue inside the persistent recovery of the search, backup and restore synchronization is used to him, here is also a network connection why I do not establish a new connection mechanism, in fact, Netty to establish a reconnect is easy to add a monitor reconnect when the connection can be, but now do not need, Well, you know,

See the Server client package file in my Code for TCP UDP service end Client

Small files send us to customize a protocol code just fine,

 *  *       *      * @ author zhuyuping        *  @version  1.0       *  @created  2014-7-15  pm 7:00:49  *  @function: Convert Files to byte-code transfer  */public class fileencoder extends messagetobyteencoder<filedatamessage>{@ Overrideprotected void encode (channelhandlercontext ctx, filedatamessage msg,bytebuf  out)  throws exception {//Protocol  //---Head Start// 1.head 2 bit  short int   2//1.sesionid 4 bit  int     4//2.hash  4 for  int         4//3.pathlenth  How many people are  ?  4//4.blenth  Number of bits  ?     4//8. start 4 bytes                &nBsp; 4//9. end 4 bytes                     4//----Body start//5.command 1 bytes            1//6. path byte[]        ? 7. byte[]             ?    msg.tobuffer (out);//out.writebytes (Msg.tobuffer ());}}

Small file decoding

Here's the file I'm using is the 2 files from the official Netty, but I've modified the next one,

large files are not due to UDP TCP bytes limit, although there is a fixlenthxxx .... Can be spliced but he still can't upload files larger than 2G,

Let's see how I got it. I covered the Netty source class and implemented the streaming upload method.

/** *  *       *      * @ author zhuyuping        *  @version  1.0       *  @created  2014-7-28  *  @function: Overwrite the source file of Netty, realize the large file segmentation   Upload  */public class MyHttpPostRequestEncoder implements ChunkedInput< Httpcontent>{ /**     * different modes to use to  encode form data.     */    public enum  EncoderMode {        /**          *  legacy mode which should work for most.  it is known to not work with oauth. for oauth use          *  {@link  encodermode#rfc3986}. the w3c form recommentations this  for submitting post form data.          */        rfc1738,        /**          * Mode which is more new  and is used for oauth         */         RFC3986    }    private  static final map<pattern, string> percentencodings = new hashmap< Pattern, string> ();    static {         percentencodings.put (Pattern.compile ("\\*"),  "%2a");         percentencodings.put (PatTern.compile ("\\+"),  "%20");         percentencodings.put ( Pattern.compile ("%7e"),  "~");    }    /**      * factory used to create interfacehttpdata     */     private final HttpDataFactory factory;    /**      * Request to encode     */     private final HttpRequest request;    /**      * Default charset to use     */     private final charset charset;    /**

Then we just send the file time, so that the original 2G is divided into join us to set up 500M is 5 minutes concurrent transmission and then in the system background splicing assembly

Above here I used messagedigest generated a checksum used to check the integrity of the file later, the background business logic is not fully written.

We use the file memory block insinuate lock form, why use it, because the file is too large if the traditional need to create in memory this very much requires a lot of memory so we use this way, the core code here, there is no add chesum check the integrity code is added later

OK then that's it, the next add RPC call is also very simple, is serialized network transport event-driven asynchronous mechanism I will use Probuffer to do serialization. You might want to make a code next week, so you don't have time to update the SQL parsing chapter so sorry, because of the text size limit

So some are pictures sorry, we can go into http://git.oschina.net/zhuyuping/distributeTemplate

or https://github.com/zhuyuping/distributeTemplate .