Common File API operations in HDFS

1. Common File API operations

 Package  CN. luxh. App. util;  Import  Java. Io. ioexception;  Import  Java. Text. simpledateformat;  Import  Java. util. date;  Import  Org. Apache. hadoop. conf. configuration;  Import  Org. Apache. hadoop. fs. blocklocation;  Import Org. Apache. hadoop. fs. fsdataoutputstream;  Import  Org. Apache. hadoop. fs. filestatus;  Import  Org. Apache. hadoop. fs. filesystem;  Import  Org. Apache. hadoop. fs. path;  Import  Org. Apache. hadoop. HDFS. distributedfilesystem;  Import  Org. Apache. hadoop. HDFS. Protocol. datanodeinfo;  Public   Class  Hdfsutil { /**  * Copy a file locally to HDFS *  @ Param  Srcfilepath *  @ Param  Dstfilepath *  @ Throws  Ioexception  */      Public   Static   Void Copyfile2hdfs (string srcfilepath, string dstfilepath) Throws  Ioexception {configuration Conf =New  Configuration (); Path SRC = New  PATH (srcfilepath); Path DST = New  PATH (dstfilepath); filesystem HDFS = DST. getfilesystem (CONF); HDFS. copyfromlocalfile (SRC, DST); filestatus [] files = HDFS. liststatus (DST );  If (Files! = Null  ){  For ( Int I = 0; I <files. length; I ++ ) {System. Out. println ( "The file is:" + Files [I]. getpath (). getname ());}}  Else  {System. Out. println ( "No files" );} HDFS. Close ();}  /**  * Create a file on HDFS *  @ Param  CONTENT *  @ Param  Dstfile * @ Throws  Ioexception  */      Public   Static   Void Createfileinhdfs (string content, string dstfile) Throws  Ioexception {configuration Conf = New  Configuration (); Path DST = New  PATH (dstfile); filesystem HDFS = Null ; Fsdataoutputstream out = Null  ;  Try  {HDFS = DST. getfilesystem (CONF); Out = HDFS. Create (DST); Out. writebytes (content); Out. Flush ();}  Catch  (Ioexception e) {e. printstacktrace ();  Throw   New  Ioexception (E );} Finally  {  If (HDFS! = Null  ) {HDFS. Close ();}  If (Out! = Null  ){  Try  {Out. Close ();}  Catch  (Ioexception e) {e. printstacktrace ();}}}} /**  * Rename a file *  @ Param  Originalfile *  @ Param  Newfile *  @ Throws  Ioexception  */      Public   Static   Void Renamefileinhdfs (string originalfile, string newfile) Throws  Ioexception {configuration Conf =New  Configuration (); Path originalpath = New  PATH (originalfile); Path newpath = New  PATH (newfile); filesystem HDFS = Newpath. getfilesystem (CONF); HDFS. Rename (originalpath, newpath); HDFS. Close ();}  /**  * Obtain the last file modification time *  @ Param  Dstfile *  @ Throws Ioexception  */      Public   Static   Void Getfilelastmodifytime (string dstfile) Throws  Ioexception {configuration Conf = New  Configuration (); Path dstpath = New  PATH (dstfile); filesystem HDFS = Dstpath. getfilesystem (CONF); filestatus File =HDFS. getfilestatus (dstpath );  Long Time = File. getmodificationtime (); HDFS. Close (); system. Out. println ( "The Last modify time is:" + New Simpledateformat ("yyyy-mm-dd hh: mm: SS"). Format ( New  Date (time )));}  /**  * Check whether the file exists *  @ Param  Dstfile *  @ Throws  Ioexception */      Public   Static   Void Checkfileisexists (string dstfile) Throws  Ioexception {configuration Conf = New  Configuration (); Path dstpath = New  PATH (dstfile); filesystem HDFS = Dstpath. getfilesystem (CONF );  Boolean Flag =HDFS. exists (dstpath); HDFS. Close (); system. Out. println ( "Is the file exists:" + Flag );}  /**  * Obtain the file storage location *  @ Param  Dstfile *  @ Throws  Ioexception  */      Public   Static   Void Getfilelocations (string dstfile) Throws Ioexception {configuration Conf = New  Configuration (); Path dstpath = New  PATH (dstfile); filesystem HDFS = Dstpath. getfilesystem (CONF); filestatus File = HDFS. getfilestatus (dstpath); blocklocation [] blklocations = HDFS. getfileblocklocations (file, 0 , File. getlen ());  If (Blklocations! = Null  ){ Int Len = Blklocations. length;  For ( Int I = 0; I <Len; I ++ ) {String [] hosts = Blklocations [I]. gethosts ();  For  (String HOST: hosts) {system. Out. println ( "The location 'host is:" + Host) ;}} HDFS. Close ();}  /**  * Delete an object * @ Throws  Ioexception  */      Public   Static   Void Deletefile (string dstfile) Throws  Ioexception {configuration Conf = New  Configuration (); Path dstpath = New  PATH (dstfile); filesystem HDFS = Dstpath. getfilesystem (CONF ); Boolean Flag = HDFS. Delete (dstpath, True  ); System. Out. println ( "Is deleted:" + Flag );}} 

 

2. Test

 Package  CN. luxh. App. test;  Import  Java. Io. ioexception;  Import  Org. JUnit. test;  Import  CN. luxh. App. util. hdfsutil; Public   Class  Hdfstester {@ Test  Public   Void Testcopyfile2hdfs () Throws  Ioexception {string srcfilepath = "/Home/coder/words.txt" ; String dstfilepath = "HDFS: // H1: 9000/user/coder/in" ; Hdfsutil. copyfile2hdfs (srcfilepath, dstfilepath);} @ Test  Public   Void Testcreatefileinhdfs () Throws Ioexception {string content = "Hey, hadoop ." ; String dstfile = "HDFS: // H1: 9000/user/coder/In/Hey" ; Hdfsutil. createfileinhdfs (content, dstfile) ;}@ Test  Public   Void Testrenamefileinhdfs () Throws  Ioexception {string originalfile = "HDFS: // H1: 9000/user/coder/In/Hey" ; String newfile = "HDFS: // H1: 9000/user/coder/In/hey_hadoop" ; Hdfsutil. renamefileinhdfs (originalfile, newfile);} @ Test Public   Void Testgetfilelastmodifytimme () Throws  Ioexception {string dstfile = "HDFS: // H1: 9000/user/coder/In/hey_hadoop" ; Hdfsutil. getfilelastmodifytime (dstfile);} @ Test  Public   Void Testcheckfileisexists () Throws  Ioexception {string dstfile = "HDFS: // H1: 9000/user/coder/In/hey_hadoop" ; Hdfsutil. checkfileisexists (dstfile);} @ Test Public   Void Testgetfilelocations () Throws  Ioexception {string dstfile = "HDFS: // H1: 9000/user/coder/In/hey_hadoop" ; Hdfsutil. getfilelocations (dstfile);} @ Test  Public   Void Testdeletefile () Throws  Ioexception {string dstfile = "HDFS: // H1: 9000/user/coder/output" ; Hdfsutil. deletefile (dstfile );}}