Access files on HDFS via JAVA-API

1. Pass The core-site.xml configuration file is configured.

Configuration item: hadoop. tmp. dir indicates the directory location where metadata is stored on the named node. For data nodes, the directory where file data is stored on the node.

Configuration item: FS. default. name indicates the IP address and port number. The default value is file: //. For Java APIs, the configured URL address must be used to connect to HDFS. For data nodes, the data node uses this URL to access the named node.

2. Use Java APIs to access and modify HDFS files.

*/** Upload files to HDFS */

 private static void uploadToHdfs() throws FileNotFoundException,IOException {  String localSrc = "d://123.txt";  String dst = "hdfs://localhost:9000/";  InputStream in = new BufferedInputStream(new FileInputStream(localSrc));  Configuration conf = new Configuration();    FileSystem fs = FileSystem.get(URI.create(dst), conf);  OutputStream out = fs.create(new Path(dst), new Progressable() {   public void progress() {    System.out.print(".");   }  });  IOUtils.copyBytes(in, out, 4096, true); }

*/** Read files from HDFS */

Private Static void uploadtohdfs () throws filenotfoundexception, ioexception {
  String localSrc = "d://123.txt";  String dst = "hdfs://localhost:9000/";
Inputstream in = new bufferedinputstream (New fileinputstream (localsrc); configuration conf = new configuration (); filesystem FS = filesystem. Get (URI. Create (DST), conf );
Outputstream out = FS. create (New Path (DST), new progressable () {public void progress () {system. out. print (". ") ;}}); ioutils. copybytes (In, out, 4096, true );}
*/** Read files from HDFS */
Private Static void readfromhdfs () throws filenotfoundexception, ioexception {
  String dst = "hdfs://localhost:9000/";
Configuration conf = new configuration (); filesystem FS = filesystem. get (URI. create (DST), conf); fsdatainputstream hdfsinstream = FS. open (New Path (DST); outputstream out = new fileoutputstream ("D:/qq-hdfs.txt"); byte [] iobuffer = new byte [1024];
Int readlen = hdfsinstream. Read (iobuffer); While (-1! = Readlen) {out. write (iobuffer, 0, readlen); readlen = hdfsinstream. read (iobuffer);} Out. close (); hdfsinstream. close (); FS. close ();}
*/** Add content in append mode to the end of the file on HDFS; Note: file update requires adding <property> <Name> DFS in the hdfs-site.xml. append. support </Name> <value> true </value> </property> */
Private Static void appendtohdfs () throws filenotfoundexception, ioexception {
   String dst = "hdfs://localhost:9000/"; 
Configuration conf = new configuration (); filesystem FS = filesystem. get (URI. create (DST), conf); fsdataoutputstream out = FS. append (New Path (DST); int readlen = "zhangzk add by HDFS Java API ". getbytes (). length; while (-1! = Readlen) {out. Write ("zhangzk add
By HDFS Java API ". getbytes (), 0, readlen);} Out. Close (); FS. Close ();}
*/** Delete an object from HDFS */
Private Static void deletefromhdfs () throws filenotfoundexception, ioexception {
  String dst = "hdfs://localhost:9000/";
Configuration conf = new configuration (); filesystem FS = filesystem. get (URI. create (DST), conf); FS. deleteonexit (New Path (DST); FS. close ();}
/** Traverse the files and directories on HDFS */
 private static void getDirectoryFromHdfs() throws FileNotFoundException,IOException {  String dst = "hdfs://localhost:9000/";   Configuration conf = new Configuration();    FileSystem fs = FileSystem.get(URI.create(dst), conf);  FileStatus fileList[] = fs.listStatus(new Path(dst));  int size = fileList.length;  for(int i = 0; i < size; i++){  System.out.println("name:" + fileList[i].getPath().getName() + "/t/tsize:" + fileList[i].getLen());  }  fs.close(); } }
* Directly set the configuration file
Configuration conf = new Configuration();conf.addResource(new Path("/project/hadoop-0.20.2/conf/hdfs-site.xml"));conf.addResource(new Path("/project/hadoop-0.20.2/conf/core-site.xml"));conf.addResource(new Path("/project/hadoop-0.20.2/conf/mapred-site.xml"));conf.addResource(new Path("/project/hadoop-0.20.2/src/hdfs/hdfs-default.xml"));conf.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, name);
ReferenceWen Mao