FileSystem instantiation Process

HDFs Case Code

New Configuration ();   = filesystem.get (New URI ("hdfs://hadoop000:8020"), configuration);     = Filesystem.open (new Path (hdfs_path+ "/hdfsapi/test/log4j.properties"New FileOutputStream (new File ("log4j_download.properties"true//  The last parameter indicates that the input/out stream is closed after the copy is completed

Filesystem.java

Static FinalCache cache =NewCache (); Public StaticFileSystem get (Uri Uri, Configuration conf)throwsIOException {String scheme= Uri.getscheme ();//HDFsString authority = uri.getauthority ();//hadoop000:8020    return Cache.get (URI, conf);} FileSystem get (Uri Uri, Configuration conf)throwsioexception{key Key=NewKey (URI, conf); return getinternal(URI, Conf, key);}PrivateFileSystemgetinternal(Uri Uri, Configuration conf, key key)throwsioexception{FileSystem FS; synchronized( This) {FS=Map.get (key); }        //a filesystem instance is obtained based on the URI, and if the cache is allowed, it is removed from the cache, otherwise the Createfilesystem is called to create a new instance    if(FS! =NULL) {         returnFS; } FS= Createfilesystem (URI, conf); synchronized( This) {FileSystem Oldfs=Map.get (key); ... //put into the cache mid-autumn        returnFS; }}Private StaticFileSystem Createfilesystem (Uri Uri, Configuration conf)throwsIOException { Class <?> clazz = Getfilesystemclass (Uri.getscheme (), conf);//returned by: Org.apache.hadoop.hdfs.DistributedFileSystemFileSystem fs =(FileSystem) reflectionutils.newinstance (clazz, conf); fs.initialize (URI, conf); //Initialize Distributedfilesystem    returnFS;} Public Staticclass<?extendsFilesystem> Getfilesystemclass (String scheme,configuration conf)throwsIOException {if(! file_systems_loaded) {//whether the file system has been loaded, false at first Loadfilesystems (); } Class<?extendsFilesystem> Clazz =NULL; if(conf! =NULL) {Clazz= (class<?extendsfilesystem>) Conf.getclass ("FS." + Scheme + ". Impl",NULL);//Fs.hdfs.impl, we did not configure this property in Core-default.xml and Core-site.xml at this time    }    if(Clazz = =NULL) { clazz = service_file_systems.get (scheme);//class Org.apache.hadoop.hdfs.DistributedFileSystem    }    if(Clazz = =NULL) {        Throw NewIOException ("No FileSystem for scheme:" +scheme); }    returnClazz;}Private Static voidLoadfilesystems () {synchronized(FileSystem.class) {        if(!file_systems_loaded) {Serviceloader<FileSystem> Serviceloader = Serviceloader.load (FileSystem.class);  for(FileSystem fs:serviceloader) {service_file_systems.put (Fs.getscheme (), Fs.getclass ()); } file_systems_loaded=true;//identified as having been loaded from the system        }    }}

After Loadfilesystems, Service_file_systems has the following values:

file=classOrg.apache.hadoop.fs.LocalFileSystem, FTP=classOrg.apache.hadoop.fs.ftp.FTPFileSystem, HDFs=classOrg.apache.hadoop.hdfs.DistributedFileSystem, Hftp=classOrg.apache.hadoop.hdfs.web.HftpFileSystem, Webhdfs=classOrg.apache.hadoop.hdfs.web.WebHdfsFileSystem, s3n=classOrg.apache.hadoop.fs.s3native.NativeS3FileSystem, Viewfs=classOrg.apache.hadoop.fs.viewfs.ViewFileSystem, Swebhdfs=classOrg.apache.hadoop.hdfs.web.SWebHdfsFileSystem, Har=classOrg.apache.hadoop.fs.HarFileSystem, S3=classOrg.apache.hadoop.fs.s3.S3FileSystem, Hsftp=classOrg.apache.hadoop.hdfs.web.HsftpFileSystem

Distributedfilesystem.java

dfsclient Dfs; //Focus Attribute: Client interaction with the server needs to get dfsclient first@Override Public voidInitialize (URI Uri, Configuration conf)throwsIOException {Super. Initialize (URI, conf);    setconf (conf); String Host= Uri.gethost ();//hadoop000    This.dfs = new Dfsclient (URI, Conf, statistics);  This. Uri = Uri.create (uri.getscheme () + "://" +uri.getauthority ());  This. Workingdir =gethomedirectory ();}

Dfsclient.java

Final ClientProtocol Namenode; //Focus attribute: RPC interface for client-to-namenode communication PublicDfsclient (URI Namenodeuri, ClientProtocol Rpcnamenode, Configuration conf, filesystem.statistics stats)throwsIOException {namenodeproxies.proxyandinfo<ClientProtocol> Proxyinfo =namenodeproxies.createproxy (conf, namenodeuri,clientprotocol.class);  This. Dtservice =Proxyinfo.getdelegationtokenservice ();  This. Namenode = Proxyinfo.getproxy ();//Org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB}

Namenodeproxies.java

 Public Static<T> proxyandinfo<t>Createproxy(Configuration conf, URI Namenodeuri, class<t> xface)throwsIOException {Class<FailoverProxyProvider<T>> Failoverproxyproviderclass =getfailoverproxyproviderclass (conf, Namenodeuri, xface); return Createnonhaproxy(Conf, namenode.getaddress (Namenodeuri), Xface,usergroupinformation.getcurrentuser (),true);} Public Static<T> proxyandinfo<t>Createnonhaproxy(Configuration conf, inetsocketaddress nnaddr, class<t>Xface, Usergroupinformation Ugi,BooleanWithretries)throwsIOException {Text dtservice=Securityutil.buildtokenservice (NNADDR);    T proxy; if(Xface = = ClientProtocol.class) {Proxy=(T) creatennproxywithclientprotocol(nnaddr, Conf, ugi,withretries); } ...    return NewProxyandinfo<t>(proxy, dtservice);}Private StaticClientProtocol Creatennproxywithclientprotocol (inetsocketaddress address, Configuration conf, Usergroupinformation Ugi,BooleanWithretries)throwsIOException {//RPC Interaction interface between client and Namenode    Final LongVersion = Rpc.getprotocolversion (CLIENTNAMENODEPROTOCOLPB.class); CLIENTNAMENODEPROTOCOLPB proxy = rpc.getprotocolproxy(CLIENTNAMENODEPROTOCOLPB.class, version, address, UGI, Conf, netutils.getdefaultsocketfactory (CONF), Org.apache.hadoop.ipc.Client.getT    Imeout (conf), Defaultpolicy). GetProxy (); if(withretries) {//create an instance using the dynamic agent of the JDKProxy =(CLIENTNAMENODEPROTOCOLPB) retryproxy.create(CLIENTNAMENODEPROTOCOLPB.class,NewDefaultfailoverproxyprovider<clientnamenodeprotocolpb>(CLIENTNAMENODEPROTOCOLPB.class, proxy), methodnametopolicymap,defaultpolicy); }    return New CLIENTNAMENODEPROTOCOLTRANSLATORPB (proxy);}

Retryproxy.java

 Public static <T> Object Create (class<t> iface,failoverproxyprovider<t> Proxyprovider, Retrypolicy retrypolicy) {    return   proxy.newproxyinstance(        Proxyprovider.getinterface (). getClassLoader (),        new class<?>[] {iface},         New retryinvocationhandler<t>(Proxyprovider, Retrypolicy)    );}

Get filesystem Instance Source Analysis Summary:

1, Filesystem.get through reflection to instantiate a distributedfilesystem;

2. Distributedfilesystem new Dfscilent () takes him as his own member variable;

3, in the Dfsclient constructs the method inside, calls the createproxy uses the RPC mechanism to obtain a Namenode proxy object, can communicate with Namenode;

4, the whole process: filesystem.get ()--distributedfilesystem.initialize ()--Dfsclient (Rpc.getprotocolproxy ())-- Namenode's agent.

FileSystem instantiation Process