Mastering the Java API Interface access for HDFS

The main purpose of the HDFs design is to store massive amounts of data, meaning that it can store a large number of files (terabytes of files can be stored). HDFs divides these files and stores them on different Datanode, and HDFs provides two access interfaces: The shell interface and the Java API interface, which operate on the files in HDFs, each of which datanode on each block. is transparent to developers.

1. Get File system

1/** 2  * Get file System 3  *  4  * @return FileSystem 5 *  /6 public static FileSystem Getfilesystem () {7     // Read Config file 8 configuration     conf = new configuration (); 9     //File system     FileSystem fs = null;11     String HDF SUri = hdfsuri;13 If     (Stringutils.isblank (Hdfsuri)) {         //Return to default file system  if running under a Hadoop cluster, You can use this method to get the default file system directly,         try {             fs = Filesystem.get (conf); C         } catch (IOException e) {             logger.error ("", e);         }20     }else{21         //Returns the specified file system, if you are testing locally, you need to use this method to get the file system             . Hdfsuri.trim ());             fs = Filesystem.get (uri,conf);         catch (URISyntaxException | IOException e) {             logger.error ("", e);         }28     }29     return fs;31}

2. Create a file directory

1/** 2  * Create file directory 3  *  4  * @param path 5 *  /6 public static void mkdir (String path) {7     try {8         Get file system 9         FileSystem fs = Getfilesystem (),         String Hdfsuri = hdfsuri;12         if ( Stringutils.isnotblank (Hdfsuri)) {             path = Hdfsuri + path;14         }15         //Create directory         Fs.mkdirs ( New path); +         //Release Resources         Fs.close (); +     } catch (IllegalArgumentException | IOException e) {         logger.error ("", e);     }24}

3. Delete files or file directories

1/** 2  * Delete file or file directory 3  *  4  * @param path 5 *  /6 public static void RmDir (String path) {7     try {8< c6/>//returns FileSystem Object 9         FileSystem fs = Getfilesystem (), and         String Hdfsuri = hdfsuri;12         if ( Stringutils.isnotblank (Hdfsuri)) {             path = Hdfsuri + path;14         }15         //delete file or file directory  Delete ( Path f) This method has been deprecated with         fs.delete (path), true); +         //Release Resources         (fs.close);     IllegalArgumentException | IOException e) {         logger.error ("", e);     }24}

3, according to the filter to obtain the file under the directory

 1/** 2 * Get directory files under Filter 3 * 4 * @param path 5 * @param pathfilter 6 * @return string[] 7 */8 public static STR Ing[] ListFile (String path,pathfilter pathfilter) {9 string[] files = new String[0];10 One-try {12// Returns the FileSystem object FileSystem fs = Getfilesystem (); String Hdfsuri = hdfsuri;16 if (stri          Ngutils.isnotblank (Hdfsuri)) {$ path = Hdfsuri + path;18}19 filestatus[] status;21 if (pathfilter! = null) {22//list Contents according to filter status = Fs.liststatus (path), PATHF         Ilter)}else{25//list contents of the Directory status = Fs.liststatus (path), 27}28         29//Get all file paths under directory path[] listedpaths = fileutil.stat2paths (status); 31//Convert STRING[]32             if (listedpaths! = null && listedpaths.length > 0) {files = new string[listedpaths.length];34 for (int i =0; i < files.length; i++) {Files[i] = listedpaths[i].tostring (); 36}37}38//Release Resources FS.C Lose (); n} catch (IllegalArgumentException | IOException e) {logger.error ("", e);}43 files;45}

4. Uploading files to HDFS

1/** 2  * file uploaded to HDFS 3  *  4  * @param delsrc 5  * @param overwrite 6  * @param srcfile 7  * @param DestPath 8  */9 public static void Copyfiletohdfs (Boolean delsrc, Boolean overwrite,string srcfile,string destpath) {1 0     //source file path is the path under Linux, if tested under Windows, it needs to be rewritten as a path under Windows, such as d://hadoop/djt/weibo.txt11     path Srcpath = new Path ( Srcfile);/     /destination path,     String Hdfsuri = hdfsuri;15     if (Stringutils.isnotblank (Hdfsuri)) {16         DestPath = Hdfsuri + destpath;17     }     path Dstpath = new Path (destpath);/     /implement file upload     try {$         //Get FileSystem Object         FileSystem fs = Getfilesystem (),         fs.copyfromlocalfile (Srcpath, Dstpath);         Fs.copyfromlocalfile (delsrc , Overwrite,srcpath, Dstpath);         //Release Resources         Fs.close (); d     } catch (IOException e) {         logger.error ("", e);     }31}

5. download files from HDFS

1/** 2  * Download file from HDFS 3  *  4  * @param srcfile 5  * @param destpath 6 */  7 public static void GetFile (String srcfile,string DestPath) {8     //source file path 9     String Hdfsuri = hdfsuri;10     if (Stringutils.isnotblank (Hdfsuri)) {one         srcfile = Hdfsuri + srcfile;12     }13 path     srcpath = new Path (srcfile); the     destination path is the path under Linux, if you test under Windows, It needs to be rewritten as a path under Windows, such as d://hadoop/djt/16     path Dstpath = new Path (destpath);         Get FileSystem object         FileSystem fs = Getfilesystem ();/         /Download the file on HDFs at         Fs.copytolocalfile (Srcpath, Dstpath);         fs.close ()/release resource (     IOException e) {         logger.error ("", e);     }28}

6. Get the HDFS cluster node information

1/** 2  * Get HDFS cluster node Information 3  *  4  * @return datanodeinfo[] 5 */  6 public static datanodeinfo[] Gethdfsnod Es () {7     //Get all Nodes 8     datanodeinfo[] datanodestats = new Datanodeinfo[0]; 9     try {One         //return filesystem Object         FileSystem fs = Getfilesystem ();         //Get Distributed File system.         Distributedfilesystem HDFs = ( Distributedfilesystem) fs;16         datanodestats = Hdfs.getdatanodestats ();     catch (IOException e) {19         Logger.error ("", e);     return datanodestats;22}

7. Find the location of a file in the HDFs cluster

1/** 2  * Find the location of a file in the HDFs cluster 3  *  4  * @param filePath 5  * @return blocklocation[] 6  */7 public stat IC blocklocation[] getfileblocklocations (string filePath) {8     ///file path 9     String Hdfsuri = hdfsuri;10     if ( Stringutils.isnotblank (Hdfsuri)) {one         FilePath = Hdfsuri + filepath;12     }13     path Path = new Path (FilePath); /     /File block location list     blocklocation[] blklocations = new blocklocation[0];17     try {         // Returns FileSystem Object         FileSystem fs = Getfilesystem ();         //Get file directory         filestatus filestatus = Fs.getfilestatus (path);         get File block Location list         blklocations = fs.getfileblocklocations (filestatus, 0, Filestatus.getlen ());     catch (IOException e) {         logger.error ("", e);     }27     return BLKLOCATIONS;28}

8. File renaming

1/** 2  * File Rename 3  *  4  * @param srcpath 5  * @param dstpath 6 *  /7 public boolean rename (String SRCP Ath, String dstpath) {8     Boolean flag = false; 9     try    {         //Return FileSystem object         FileSystem fs = GetFileS Ystem ();         String Hdfsuri = hdfsuri;14         if (Stringutils.isnotblank (Hdfsuri)) {             Srcpath = Hdfsuri + srcpath;16             dstpath = Hdfsuri + dstpath;17         }18         flag = fs.rename (new Path (Srcpath), new Pa Th (Dstpath)); (     IOException e) {         logger.error ("{} rename to {} error.", Srcpath, Dstpath);     return flag;25}

9. Determine if the directory exists

1/** 2  * Determine if the directory exists 3  *  4  * @param srcpath 5  * @param dstpath 6 *  /7 public boolean Existdir (String FilePath, Boolean Create) {8     Boolean flag = False, 9     if (Stringutils.isempty (FilePath)) {One         return flag;12     }13     try{15         path Path = new Path (filePath), +         //FileSystem object         FileSystem FS = Getfilesystem (),         if (         create) {             !fs.exists (path)) {                 fs.mkdirs (path);             }23         }24         if (fs.isdirectory (path)) {$             flag = true;27         }28     }catch (Exception e) {         Logger.error ("", e);     }31     return flag;33}

View the last modified time for the HDFs file

2. Public void Testgetmodifytime () throws Exception {
4. Configuration conf = new configuration ();
6. FileSystem HDFs = filesystem.get (conf);
8. Path DST = New Path (Hdfspath);
10. Filestatus files[] = hdfs.liststatus (DST);
12. for (Filestatus file:files) {
14. System.out.println (File.getpath () + "\ T"
16. + File.getmodificationtime ());
18. System.out.println (File.getpath () + "\ T"
20. + New Date (File.getmodificationtime ()));
22. }

2. //See if the HDFs file exists
6. public void Testexists () throws Exception {
10. Configuration conf = new configuration ();
14. FileSystem HDFs = filesystem.get (conf);
16. Path DST = New Path (Hdfspath + "File01.txt");
20. Boolean OK = hdfs.exists (DST);
22. System.out.println (ok?)  " file exists": " file does not exist");
24. }

2. //Get all node names on the HDFs cluster
4. public void Testgethostname () throws Exception {
8. Configuration conf = new configuration ();
12. Distributedfilesystem HDFs = (distributedfilesystem) FileSystem
14. . get (conf);
16. datanodeinfo[] Datanodestats = Hdfs.getdatanodestats ();
20. For (Datanodeinfo datanode:datanodestats) {
22. System.out.println (Datanode.gethostname () + "\ T"
24. + Datanode.getname ());
26. }
28. }

Mastering the Java API Interface access for HDFS