Reading information on a Hadoop cluster using the HDFS client Java API

This article describes the configuration method for using the HDFs Java API.

1, first solve the dependence, pom

<Dependency>            <groupId>Org.apache.hadoop</groupId>            <Artifactid>Hadoop-client</Artifactid>            <version>2.7.2</version>            <Scope>Provided</Scope>        </Dependency>

2, configuration files, storage HDFs cluster configuration information, basically from Core-site.xml and Hdfs-site.xml, can be based on the HDFS cluster client configuration file information to fill in

#============== Hadoop ===================hdfs.fs.defaultFS=hdfs://MYCLUSTER-TJ hdfs.ha.zookeeper.quorum=xxxx-apache00.xx01,xxxx-apache01.xx01,xxxx-apache02. XX01hdfs.dfs.nameservices=xxxxhdfs.dfs.ha.namenodes.mycluster-tj=xx1, XX2hdfs.dfs.namenode.rpc-address.mycluster-tj.nn1=xxxx-apachenn01.xx01:8020  Hdfs.dfs.namenode.rpc-address.mycluster-tj.nn2=xxxx-apachenn02.xx01:8020

3. Java Client API

Importjava.io.IOException;ImportJava.net.URI;ImportOrg.apache.hadoop.fs.FileStatus;ImportOrg.apache.hadoop.fs.FileSystem;ImportOrg.apache.hadoop.fs.Path;Importorg.apache.hadoop.conf.Configuration;ImportOrg.slf4j.Logger;Importorg.slf4j.LoggerFactory;ImportJava.text.SimpleDateFormat;Importjava.util.Date; Public classhadoopclient {protected FinalLogger Logger = Loggerfactory.getlogger ( This. GetClass ()); PrivateFileSystem FS; PrivateString Defaultfs; PrivateString Zkquorum; PrivateString nameservices; PrivateString namenodes; PrivateString rpcAddressNN1; PrivateString rpcAddressNN2;  Public voidSetdefaultfs (String defaultfs) { This. Defaultfs =Defaultfs; }     PublicString Getdefaultfs () {returnDefaultfs; }     Public voidSetzkquorum (String zkquorum) { This. Zkquorum =Zkquorum; }     PublicString Getzkquorum () {returnZkquorum; }     Public voidsetnameservices (String nameservices) { This. nameservices =nameservices; }     PublicString getnameservices () {returnnameservices; }     Public voidsetnamenodes (String namenodes) { This. Namenodes =Namenodes; }     PublicString getnamenodes () {returnNamenodes; }     Public voidsetRpcAddressNN1 (String rpcAddressNN1) { This. rpcAddressNN1 =rpcAddressNN1; }     PublicString getRpcAddressNN1 () {returnrpcAddressNN1; }     Public voidsetRpcAddressNN2 (String rpcAddressNN2) { This. rpcAddressNN2 =rpcAddressNN2; }     PublicString getRpcAddressNN2 () {returnrpcAddressNN2; }     Public voidinit () {Try{Configuration conf=NewConfiguration (); Conf.set ("Fs.defaultfs", Defaultfs); Conf.set ("Ha.zookeeper.quorum", Zkquorum); Conf.set ("Dfs.nameservice", nameservices); Conf.set ("DFS.HA.NAMENODES.MYCLUSTER-TJ", Namenodes); Conf.set ("Dfs.namenode.rpc-address.mycluster-tj.nn1", rpcAddressNN1); Conf.set ("Dfs.namenode.rpc-address.mycluster-tj.nn2", rpcAddressNN2); FS= Filesystem.get (NewURI (DEFAULTFS), conf); } Catch(Exception ex) {ex.printstacktrace (); }    }     Public voidStop () {Try{fs.close (); } Catch(Exception e) {}} Public Booleanexists (String path) {BooleanIsexists =false; Try{Path Hdfspath=Newpath (path); Isexists=fs.exists (Hdfspath); } Catch(Exception ex) {Logger.error ("Exists error: {}", Ex.getmessage ()); }        returnisexists; }     PublicString Getmodificationtime (String path)throwsIOException {String modifytime=NULL; Try{Path Hdfspath=Newpath (path); Filestatus Filestatus=Fs.getfilestatus (Hdfspath); LongModifytimestamp =Filestatus.getmodificationtime (); SimpleDateFormat SimpleDateFormat=NewSimpleDateFormat ("Yyyymmddhhmmss"); Date Date=NewDate (Modifytimestamp); Modifytime=Simpledateformat.format (date); } Catch(Exception ex) {Logger.error ("Getmodificationtime error: {}", Ex.getmessage ()); }        returnModifytime; }}

4. Configuration

Importcom.xiaoju.dqa.prometheus.client.hadoop.HadoopClient;ImportOrg.springframework.beans.factory.annotation.Value;ImportOrg.springframework.context.annotation.Bean;Importorg.springframework.context.annotation.Configuration; @Configuration Public classhadoopconfiguration {@Value ("${hdfs.fs.defaultfs}")    PrivateString Defaultfs; @Value ("${hdfs.ha.zookeeper.quorum}")    PrivateString Zkquorum; @Value ("${hdfs.dfs.nameservices}")    PrivateString nameservices; @Value ("${HDFS.DFS.HA.NAMENODES.MYCLUSTER-TJ}")    PrivateString namenodes; @Value ("${hdfs.dfs.namenode.rpc-address.mycluster-tj.nn1}")    PrivateString rpcAddressNN1; @Value ("${hdfs.dfs.namenode.rpc-address.mycluster-tj.nn2}")    PrivateString rpcAddressNN2; @Bean (Initmethod= "Init", Destroymethod = "Stop")     Publichadoopclient hadoopclient () {hadoopclient hadoopclient=Newhadoopclient ();        Hadoopclient.setdefaultfs (DEFAULTFS);        Hadoopclient.setzkquorum (Zkquorum);        Hadoopclient.setnameservices (nameservices);        Hadoopclient.setnamenodes (Namenodes);        HADOOPCLIENT.SETRPCADDRESSNN1 (rpcAddressNN1);        HADOOPCLIENT.SETRPCADDRESSNN2 (rpcAddressNN2); returnhadoopclient; }}

Reading information on a Hadoop cluster using the HDFS client Java API