HDFS JAVA代碼進行增刪改查

提醒：如果要在idear或者eclipse等IDE下運行就必須在HDFS上給你所使用的目錄分配許可權給windows下的使用者，為了方便起見建議給擁有權限777

創建目錄命令

hdfs dfs -mkdir myproject


分配許可權命令 hdfs dfs -chmod 777 myproject HDFS增刪改查工具類 package hdfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.IOException; import java.util.ArrayList; import java.util.List; /** * Created by LiuwenSheng on 2017/8/7. */ public class HDFSUtils { private final String FILE_PROTOCOL = "file:///"; private final String HDFS_PROTOCOL = "hdfs:// xxx:9000/myproject";//此處xxx表示你的IP位址比如：192.168.18.1 /** * 用於從本地檔中選擇檔上傳至hdfs中 * @param conf org.apache.hadoop.conf.Configuration 配置資訊 * @param localPath String 本地檔的存儲路徑 * @param hdfsPath String 上傳到HDFS的路徑 eg:/a/b. txt * @throws IOException */ public void file2HDFS( Configuration conf,String localPath,String hdfsPath)throws IOException { FileSystem fs = FileSystem.get(conf); fs.copyFromLocalFile(new Path(FILE_PROTOCOL+localPath),new Path(HDFS_PROTOCOL +hdfsPath)); fs.close(); } /** * 測試檔案是否存在 * @param conf 配置資訊 * @param path 檔路徑 * @return * @throws IOException */ public boolean isExit(Configuration conf,String path)throws IOException{ FileSystem fs = FileSystem.get(conf); boolean exit = fs.exists(new Path(HDFS_PROTOCOL+path)); fs.close(); return exit; } /** * 刪除檔 * @param conf org.apache.hadoop.conf.Configuration 配置資訊 * @param path 要刪除的檔路徑 * @throws IOException */ publi c Boolean deleteFormHDFS(Configuration conf,String path) throws IOException { FileSystem fs = FileSystem.get(conf); boolea n isDel = fs.deleteOnExit(new Path(HDFS_PROTOCOL+path)); fs.close(); return isDel; } /** * 把HDFS上面的檔下載到本地 * @param conf 配置資訊 * @param hdfsPath hdfs的檔路徑 * @param localPath 本地檔路徑 * @throws IOException */ public void file2Loacl(Configuration conf,String hdfsPath,String localPath)throws IOException{ FileSystem fs = FileSystem.get(conf); fs.copyToLocalFile(new Path(HDFS_PROTOCOL+hdfsPath),new Path(FILE_PROTOCOL+localPath)); fs.close(); } /** * 創建hdfs目錄 * @param conf 配置資訊 * @param path 需要創建的目錄路徑 * @return 是否創建成功 * @throws IOException */ public Boolean mkHDFSDir(Configuration conf,String path)throws IOException{ FileSystem fs = FileSystem.get(conf ); Path mypath = new Path(HDFS_PROTOCOL + path); boolean ismk = false; if(!fs.exists(mypath)) { ismk = fs.mkdirs(mypath); } fs.close(); return ismk; } /** * 查看目前的目錄內檔案清單 * @param conf 配置資訊 * @param path 查看的目錄 * @return 檔狀態陣列 * @throws IOException */ public List lsHDFSDir (Configuration conf, String path) throws IOException{ List list = new ArrayList(); FileSystem fs = FileSystem.get(conf); FileStatus[] fileStatuses = fs.listStatus(new Path(HDFS_PROTOCOL + path)); for (FileStatus fileStatus:fileStatuses){ list.add(fileStatus.getPath().toString()); } fs.close(); return list; } } 測試案例 package test; import hdfs. HDFSUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.junit.Before; import org.junit.Test; import javax.ws.rs.core.CoNtext; import java.io.IOException; import java.util.List; /** * Created by lws on 2017/8/7. */ public class MyHDFSTest { private HDFSUtils hdfsUtils = new HDFSUtils(); private Configuration conf = new Configuration(); @Before public void beforeBegining(){ conf.set("fs.defaultFS", "hdfs://xxx:9000/");//此處xxx為你的IP位址，如192.168.12.8 } @ Test public void test_file2HDFS()throws IOException{ hdfsUtils.file2HDFS(conf,"D:\\BugReport.txt","/myuser2/1"); } @ Test public void test_deleteFormHDFS()throws IOException{ hdfsUtils.deleteFormHDFS(conf,"/myuser/BugReport.txt"); } @ Test public void test_isExit()throws IOException{ boolean is = hdfsUtils.isExit(conf,"/myuser/BugReport.txt"); System.out.print(is); } @Test public void test_file2Local()throws IOException{ hdfsUtils.file2Loacl(conf,"/myuser/BugReport.txt","D://test "); } @Test public void test_mkHDFSDir()throws IOException{ boolean a = hdfsUtils.mkHDFSDir(conf,"/myuser/liming"); System.out.print(a); } @Test public void test_lsHDFSDir()throws IOException{ List list = hdfsUtils.lsHDFSDir(conf,"/myuser2"); for (String file :list){ System.out.println(file); } } }