The main purpose of the HDFs design is to store massive amounts of data, meaning that it can store a large number of files (terabytes of files can be stored). HDFs divides these files and stores them on different Datanode, and HDFs provides two access interfaces: The shell interface and the Java API interface, which operate on the files in HDFs, each of which datanode on each block. is transparent to developers.
1. Get File system
1/** 2 * Get file System 3 * 4 * @return FileSystem 5 * /6 public static FileSystem Getfilesystem () {7 // Read Config file 8 configuration conf = new configuration (); 9 //File system FileSystem fs = null;11 String HDF SUri = hdfsuri;13 If (Stringutils.isblank (Hdfsuri)) { //Return to default file system if running under a Hadoop cluster, You can use this method to get the default file system directly, try { fs = Filesystem.get (conf); C } catch (IOException e) { logger.error ("", e); }20 }else{21 //Returns the specified file system, if you are testing locally, you need to use this method to get the file system . Hdfsuri.trim ()); fs = Filesystem.get (uri,conf); catch (URISyntaxException | IOException e) { logger.error ("", e); }28 }29 return fs;31}
2. Create a file directory
1/** 2 * Create file directory 3 * 4 * @param path 5 * /6 public static void mkdir (String path) {7 try {8 Get file system 9 FileSystem fs = Getfilesystem (), String Hdfsuri = hdfsuri;12 if ( Stringutils.isnotblank (Hdfsuri)) { path = Hdfsuri + path;14 }15 //Create directory Fs.mkdirs ( New path); + //Release Resources Fs.close (); + } catch (IllegalArgumentException | IOException e) { logger.error ("", e); }24}
3. Delete files or file directories
1/** 2 * Delete file or file directory 3 * 4 * @param path 5 * /6 public static void RmDir (String path) {7 try {8< c6/>//returns FileSystem Object 9 FileSystem fs = Getfilesystem (), and String Hdfsuri = hdfsuri;12 if ( Stringutils.isnotblank (Hdfsuri)) { path = Hdfsuri + path;14 }15 //delete file or file directory Delete ( Path f) This method has been deprecated with fs.delete (path), true); + //Release Resources (fs.close); IllegalArgumentException | IOException e) { logger.error ("", e); }24}
3, according to the filter to obtain the file under the directory
1/** 2 * Get directory files under Filter 3 * 4 * @param path 5 * @param pathfilter 6 * @return string[] 7 */8 public static STR Ing[] ListFile (String path,pathfilter pathfilter) {9 string[] files = new String[0];10 One-try {12// Returns the FileSystem object FileSystem fs = Getfilesystem (); String Hdfsuri = hdfsuri;16 if (stri Ngutils.isnotblank (Hdfsuri)) {$ path = Hdfsuri + path;18}19 filestatus[] status;21 if (pathfilter! = null) {22//list Contents according to filter status = Fs.liststatus (path), PATHF Ilter)}else{25//list contents of the Directory status = Fs.liststatus (path), 27}28 29//Get all file paths under directory path[] listedpaths = fileutil.stat2paths (status); 31//Convert STRING[]32 if (listedpaths! = null && listedpaths.length > 0) {files = new string[listedpaths.length];34 for (int i =0; i < files.length; i++) {Files[i] = listedpaths[i].tostring (); 36}37}38//Release Resources FS.C Lose (); n} catch (IllegalArgumentException | IOException e) {logger.error ("", e);}43 files;45}
4. Uploading files to HDFS
1/** 2 * file uploaded to HDFS 3 * 4 * @param delsrc 5 * @param overwrite 6 * @param srcfile 7 * @param DestPath 8 */9 public static void Copyfiletohdfs (Boolean delsrc, Boolean overwrite,string srcfile,string destpath) {1 0 //source file path is the path under Linux, if tested under Windows, it needs to be rewritten as a path under Windows, such as d://hadoop/djt/weibo.txt11 path Srcpath = new Path ( Srcfile);/ /destination path, String Hdfsuri = hdfsuri;15 if (Stringutils.isnotblank (Hdfsuri)) {16 DestPath = Hdfsuri + destpath;17 } path Dstpath = new Path (destpath);/ /implement file upload try {$ //Get FileSystem Object FileSystem fs = Getfilesystem (), fs.copyfromlocalfile (Srcpath, Dstpath); Fs.copyfromlocalfile (delsrc , Overwrite,srcpath, Dstpath); //Release Resources Fs.close (); d } catch (IOException e) { logger.error ("", e); }31}
5. download files from HDFS
1/** 2 * Download file from HDFS 3 * 4 * @param srcfile 5 * @param destpath 6 */ 7 public static void GetFile (String srcfile,string DestPath) {8 //source file path 9 String Hdfsuri = hdfsuri;10 if (Stringutils.isnotblank (Hdfsuri)) {one srcfile = Hdfsuri + srcfile;12 }13 path srcpath = new Path (srcfile); the destination path is the path under Linux, if you test under Windows, It needs to be rewritten as a path under Windows, such as d://hadoop/djt/16 path Dstpath = new Path (destpath); Get FileSystem object FileSystem fs = Getfilesystem ();/ /Download the file on HDFs at Fs.copytolocalfile (Srcpath, Dstpath); fs.close ()/release resource ( IOException e) { logger.error ("", e); }28}
6. Get the HDFS cluster node information
1/** 2 * Get HDFS cluster node Information 3 * 4 * @return datanodeinfo[] 5 */ 6 public static datanodeinfo[] Gethdfsnod Es () {7 //Get all Nodes 8 datanodeinfo[] datanodestats = new Datanodeinfo[0]; 9 try {One //return filesystem Object FileSystem fs = Getfilesystem (); //Get Distributed File system. Distributedfilesystem HDFs = ( Distributedfilesystem) fs;16 datanodestats = Hdfs.getdatanodestats (); catch (IOException e) {19 Logger.error ("", e); return datanodestats;22}
7. Find the location of a file in the HDFs cluster
1/** 2 * Find the location of a file in the HDFs cluster 3 * 4 * @param filePath 5 * @return blocklocation[] 6 */7 public stat IC blocklocation[] getfileblocklocations (string filePath) {8 ///file path 9 String Hdfsuri = hdfsuri;10 if ( Stringutils.isnotblank (Hdfsuri)) {one FilePath = Hdfsuri + filepath;12 }13 path Path = new Path (FilePath); / /File block location list blocklocation[] blklocations = new blocklocation[0];17 try { // Returns FileSystem Object FileSystem fs = Getfilesystem (); //Get file directory filestatus filestatus = Fs.getfilestatus (path); get File block Location list blklocations = fs.getfileblocklocations (filestatus, 0, Filestatus.getlen ()); catch (IOException e) { logger.error ("", e); }27 return BLKLOCATIONS;28}
8. File renaming
1/** 2 * File Rename 3 * 4 * @param srcpath 5 * @param dstpath 6 * /7 public boolean rename (String SRCP Ath, String dstpath) {8 Boolean flag = false; 9 try { //Return FileSystem object FileSystem fs = GetFileS Ystem (); String Hdfsuri = hdfsuri;14 if (Stringutils.isnotblank (Hdfsuri)) { Srcpath = Hdfsuri + srcpath;16 dstpath = Hdfsuri + dstpath;17 }18 flag = fs.rename (new Path (Srcpath), new Pa Th (Dstpath)); ( IOException e) { logger.error ("{} rename to {} error.", Srcpath, Dstpath); return flag;25}
9. Determine if the directory exists
1/** 2 * Determine if the directory exists 3 * 4 * @param srcpath 5 * @param dstpath 6 * /7 public boolean Existdir (String FilePath, Boolean Create) {8 Boolean flag = False, 9 if (Stringutils.isempty (FilePath)) {One return flag;12 }13 try{15 path Path = new Path (filePath), + //FileSystem object FileSystem FS = Getfilesystem (), if ( create) { !fs.exists (path)) { fs.mkdirs (path); }23 }24 if (fs.isdirectory (path)) {$ flag = true;27 }28 }catch (Exception e) { Logger.error ("", e); }31 return flag;33}
View the last modified time for the HDFs file
- Public void Testgetmodifytime () throws Exception {
- Configuration conf = new configuration ();
- FileSystem HDFs = filesystem.get (conf);
- Path DST = New Path (Hdfspath);
- Filestatus files[] = hdfs.liststatus (DST);
- for (Filestatus file:files) {
- System.out.println (File.getpath () + "\ T"
- + File.getmodificationtime ());
- System.out.println (File.getpath () + "\ T"
- + New Date (File.getmodificationtime ()));
- }
- //See if the HDFs file exists
- public void Testexists () throws Exception {
- Configuration conf = new configuration ();
- FileSystem HDFs = filesystem.get (conf);
- Path DST = New Path (Hdfspath + "File01.txt");
- Boolean OK = hdfs.exists (DST);
- System.out.println (ok?) " file exists": " file does not exist");
- }
- //Get all node names on the HDFs cluster
- public void Testgethostname () throws Exception {
- Configuration conf = new configuration ();
- Distributedfilesystem HDFs = (distributedfilesystem) FileSystem
- . get (conf);
- datanodeinfo[] Datanodestats = Hdfs.getdatanodestats ();
- For (Datanodeinfo datanode:datanodestats) {
- System.out.println (Datanode.gethostname () + "\ T"
- + Datanode.getname ());
- }
- }
Mastering the Java API Interface access for HDFS