通過JAVA—API訪問HDFS 上的檔案

1.  通過對 core-site.xml設定檔進行配置。

   配置項：hadoop.tmp.dir表示命名節點上存放中繼資料的目錄位置，對於資料節點則為該節點上存放檔案資料的目錄。   

   配置項：fs.default.name表示命名的IP地址和連接埠號碼,預設值是file:///，對於JavaAPI來講，串連HDFS必須使用這裡的配置的URL地址，對於資料節點來講，資料節點通過該URL來訪問命名節點。

2 利用java api來訪問hdfs上的檔案，並進行修改。

    * /**上傳檔案到HDFS上去*/

 private static void uploadToHdfs() throws FileNotFoundException,IOException {  String localSrc = "d://123.txt";  String dst = "hdfs://localhost:9000/";  InputStream in = new BufferedInputStream(new FileInputStream(localSrc));  Configuration conf = new Configuration();    FileSystem fs = FileSystem.get(URI.create(dst), conf);  OutputStream out = fs.create(new Path(dst), new Progressable() {   public void progress() {    System.out.print(".");   }  });  IOUtils.copyBytes(in, out, 4096, true); }

  * /**從HDFS上讀取檔案*/

 private static void uploadToHdfs() throws FileNotFoundException,IOException { 
  String localSrc = "d://123.txt";  String dst = "hdfs://localhost:9000/";
 InputStream in = new BufferedInputStream(new FileInputStream(localSrc)); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(dst), conf);
 OutputStream out = fs.create(new Path(dst), new Progressable() { public void progress() { System.out.print("."); } }); IOUtils.copyBytes(in, out, 4096, true); }
  * /**從HDFS上讀取檔案*/
private static void readFromHdfs() throws FileNotFoundException,IOException {
  String dst = "hdfs://localhost:9000/";
 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(dst), conf); FSDataInputStream hdfsInStream = fs.open(new Path(dst)); OutputStream out = new FileOutputStream("d:/qq-hdfs.txt"); byte[] ioBuffer = new byte[1024];
 int readLen = hdfsInStream.read(ioBuffer); while(-1 != readLen){ out.write(ioBuffer, 0, readLen); readLen = hdfsInStream.read(ioBuffer); } out.close(); hdfsInStream.close(); fs.close(); }
 * /**以append方式將內容添加到HDFS上檔案的末尾;注意：檔案更新，需要在hdfs-site.xml中添<property><name>dfs.append.support</name><value>true</value></property>*/
private static void appendToHdfs() throws FileNotFoundException,IOException {  
   String dst = "hdfs://localhost:9000/"; 
 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(dst), conf); FSDataOutputStream out = fs.append(new Path(dst)); int readLen = "zhangzk add by hdfs java api".getBytes().length; while(-1 != readLen){ out.write("zhangzk add
 by hdfs java api".getBytes(), 0, readLen); } out.close(); fs.close(); }
 * /**從HDFS上刪除檔案*/
 private static void deleteFromHdfs() throws FileNotFoundException,IOException {  
  String dst = "hdfs://localhost:9000/";
 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(dst), conf); fs.deleteOnExit(new Path(dst)); fs.close(); }
 /**遍曆HDFS上的檔案和目錄*/
 private static void getDirectoryFromHdfs() throws FileNotFoundException,IOException {  String dst = "hdfs://localhost:9000/";   Configuration conf = new Configuration();    FileSystem fs = FileSystem.get(URI.create(dst), conf);  FileStatus fileList[] = fs.listStatus(new Path(dst));  int size = fileList.length;  for(int i = 0; i < size; i++){  System.out.println("name:" + fileList[i].getPath().getName() + "/t/tsize:" + fileList[i].getLen());  }  fs.close(); } }
 * 最後直接設定設定檔
Configuration conf = new Configuration();conf.addResource(new Path("/project/hadoop-0.20.2/conf/hdfs-site.xml"));conf.addResource(new Path("/project/hadoop-0.20.2/conf/core-site.xml"));conf.addResource(new Path("/project/hadoop-0.20.2/conf/mapred-site.xml"));conf.addResource(new Path("/project/hadoop-0.20.2/src/hdfs/hdfs-default.xml"));conf.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, name);
參考文茂