IDEA 建立HDFS項目 JAVA api

標籤：stream   cto   conf   throws   setup   提醒   .net   hdf   open   

1.建立quickMaven

1.在properties中寫hadoop 的版本號碼並且通過EL運算式的方式映射到dependency中

2.寫一個repostory將依賴載入到本地倉庫中

這是載入完成的頁面

 

 

 

 

這是開發代碼

package com.kevin.hadoop;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;

/**
 * Created by Administrator on 2018/7/21 0021.
 * Hadoop HDFS java API 操作
 */
public class HDFSApp {
    public static final String HDFS_PATH = "hdfs://hadoop000:8020";
    //檔案系統
    FileSystem fileSystem = null;
    //配置類
    Configuration configuration = null;

    //Before適用於類載入之前
    @Test
    /**
     * 建立HDFS系統
     */
    public void mkdir() throws Exception {
        fileSystem.mkdirs(new Path("/hdfsapi/test"));
    }

    @Test
    public void create() throws Exception {
        FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/test/a.txt"));
        output.write("hello hadoop".getBytes());
        output.flush();
        output.close();
    }

    @Test
    /**
     * 查看HDFS檔案上的內容
     */
    public void cat() throws Exception {
        FSDataInputStream in = fileSystem.open(new Path("/hdfsapi/test/a.txt"));
        IOUtils.copyBytes(in, System.out, 1024);
        in.close();
    }

    @Before
    public void setUp() throws Exception {
        System.out.printf("HDFSapp.setup");
        configuration = new Configuration();
        //拿到檔案系統
        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "hadoop");
    }

    //關閉資源用的這個
    @After
    public void tearDown() throws Exception {
        //釋放資源
        configuration = null;
        fileSystem = null;
        System.out.printf("HDFSAPP.tearDown");
    }

    /**
     * 重新命名檔案
     */
    @Test
    public void rename() throws Exception {
        Path oldPath = new Path("/hdfsapi/test/a.txt");
        Path newPath = new Path("/hdfsapi/test/b.txt");
        fileSystem.rename(oldPath, newPath);
    }

    /**
     * 上傳一個檔案
     *
     * @throws Exception
     */
    @Test
    public void copyFromLocalFile() throws Exception {
        Path localPath = new Path(" //c//hello.txt");
        Path hdfsPath = new Path("/hdfsapi/test");
        fileSystem.copyFromLocalFile(localPath, hdfsPath);
    }

    /**
     * 上傳一個大檔案
     *
     * @throws Exception
     */
    @Test
    public void copyFromLocalBigFile() throws Exception {
        InputStream in = new BufferedInputStream(
                new FileInputStream(
                        new File("C:\\mmall-fe\\all.zip")));

        FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/test/all.zip"),
                new Progressable() {
                    public void progress() {
                        System.out.print(".");  //帶進度提醒資訊
                    }
                });


        IOUtils.copyBytes(in, output, 4096);
    }
    /**
     * 下載HDFS檔案
     */
    @Test
    public void copyTOLocalFile() throws Exception{
        Path localPath = new Path("C:\\test\\b.txt");
        Path hdfsPath = new Path("/hdfsapi/test/hello.txt");
        fileSystem.copyToLocalFile(false,hdfsPath,localPath,true);
    }
    @Test
    public void listFiles() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/hdfsapi/test"));

        for (FileStatus fileStatus : fileStatuses) {
            String isDir = fileStatus.isDirectory() ? "檔案夾" : "檔案";
            //副本
            short replication = fileStatus.getReplication();
            //大小
            long len = fileStatus.getLen();
            //路徑
            String path = fileStatus.getPath().toString();

            System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path);
        }
    }

@Test
public void delete() throws Exception{
    fileSystem.delete(new Path("hdfsapi/test/"),true);
}

}


注意在查詢檔案的副本係數時如果是通過hadoop sell put上去的檔案副本係數在偽分布式中是1，如果說是通過java api put上去的則副本係數是1,因為我們並沒有在本地設定自己的副本係數，所以hadoop採用自己的副本係數

IDEA 建立HDFS項目 JAVA api