HDFS Javaapi Operation __java

A. Configure the Hadoop development environment under Windows

1. Download Hadoop and Windows version of Winutils

Download the Hadoop compression Pack (the author is using version 2.7.3)
URL: http://hadoop.apache.org/releases.html

Download the Windows version of Winutils (for 2.7.x)
URL: Https://github.com/SweetInk/hadoop-common-2.7.1-bin

Unzip Hadoop and Winutils, and overwrite all files extracted winutils with the Hadoop bin directory
Copy the Hadoop.dll file in the Winutils directory to the C:\windows\system32 directory

2. Configure Hadoop Environment Variables

New system variable, hadoop_home:f:\software\hadoop-2.7.3
Join Path,;%hadoop_home%\bin;
Finally reboot computer Two, create Maven project

I use the development tool for idea, you can also use Eclipse or other development tools to create a MAVEN project.

The 1.pom.xml file is configured as follows

<?xml version= "1.0" encoding= "UTF-8"?> <project xmlns= "http://maven.apache.org/POM/4.0.0" xmlns:xsi= "HTT" P://www.w3.org/2001/xmlschema-instance "xsi:schemalocation=" http://maven.apache.org/POM/4.0.0 Http://maven.apach E.org/xsd/maven-4.0.0.xsd "> <modelVersion>4.0.0</modelVersion> <groupid>com.bigdata</gro upid> <artifactId>hadoop-hdfs-javaapi</artifactId> <version>1.0</version> <depe ndencies> <dependency> <groupId>org.apache.hadoop</groupId> <artif
        Actid>hadoop-client</artifactid> <version>2.7.3</version> </dependency> <dependency> <groupId>junit</groupId> <artifactid>junit</artifacti d> <version>4.10</version> <scope>test</scope> </dependency&
    Gt </dependencies&Gt <build> <plugins> <plugin> &LT;GROUPID&GT;ORG.APACHE.MAVEN.PLUGINS&L T;/groupid> <artifactId>maven-compiler-plugin</artifactId> <version>3
                    .5.1</version> <configuration> <source>1.8</source> <target>1.8</target> </configuration> </plugin> <
 /plugins> </build> </project>

2. Configure Log4j.properties

Under Src/main/resource, create a new file Log4j.properties and configure the following

#OFF, Systemout,logfile,logdailyfile,logrollingfile,logmail,logdb,all
log4j.rootlogger=all,systemout

# Output to console
log4j.appender.systemout= org.apache.log4j.ConsoleAppender 
log4j.appender.systemout.layout= Org.apache.log4j.PatternLayout 
log4j.appender.systemout.layout.conversionpattern= [%-5p][%-22d{yyyy/mm/dd HH : mm:sss}][%l]%n%m%n 
log4j.appender.systemout.threshold= INFO 
log4j.appender.systemout.immediateflush= TRUE 

Finally, copy and paste five profiles of Hadoop into the src\main\resources directory

iii. Java API operation HDFs

Client to operate HDFs, need to have a user identity, by default, the HDFs client API takes a parameter from the JVM as its own user identity, there are several ways to configure the user, as follows

(1). JVM Add

Click on the top left corner to select the first item

Add Content:-dhadoop_user_name=hadoop

(2) Configure environment variables

Hadoop_user_name
Hadoop

Java API Operations

1. File copy, directory creation and deletion

Package Com.bigdata.hdfs;
Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.apache.log4j.Logger;
Import Org.junit.After;
Import Org.junit.Before;

Import Org.junit.Test;
Import Java.io.File;
Import Java.net.URI;
Import Java.util.Iterator;

Import Java.util.Map.Entry;

    public class Testhdfs {public static Logger Logger = Logger.getlogger (Testhdfs.class);
    public filesystem fs = null;

    public Configuration conf = null; @Before public void Init () throws exception{//Create a Hadoop configuration context object to load the default configuration of Hadoop at run time conf = new Configu

        Ration ();
    Create Hadoop file System object, execute Hadoop file Operation fs = Filesystem.get (conf); //Copy Local file upload to HDFs @Test public void Testuploadfile () throws Exception {fs.copyfromlocalfile (new Pat
    H ("C:/hdfstest.txt"), New Path ("/hdfstest.txt"));
        ///Copy files from HDFs to local @Test public void Testdownloadfile () throws Exception {Fs.copytolocalfile (New Path ("/hdfstest.txt"), New Path ("d:/"); }//Create directory @Test public void Testmakedir () throws Exception {Boolean flag = Fs.mkdirs (new Path ("/a/b/c
       "));
       if (flag) {Logger.info ("Directory creation successful!"); }//delete directory @Test public void Testrmovedir () throws Exception {Boolean flag = Fs.delete (New Path
        ("/A"), true);//recursive Delete if (flag) {logger.info ("Directory deletion succeeded!"); }//Close resource @After public void Destory () throws Exception {if (fs!= null) {Fs.close
        (); }
    }
}

2. File Bulk upload

Upload all files in the project upload directory to the HDFs user directory (/home/hadoop), upload directory structure as follows

Package Com.bigdata.hdfs;
Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.FSDataOutputStream;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.apache.hadoop.io.IOUtils;
Import Org.apache.log4j.Logger;

Import Org.junit.Test;
Import Java.io.File;
Import Java.io.FileInputStream;

Import Java.io.InputStream;
    public class Testuploadhdfs {public static Logger Logger = Logger.getlogger (Testuploadhdfs.class);
    public filesystem fs = null;

    public Configuration conf = null;

        @Test public void Testupload () throws exception{conf = new Configuration ();

        FS = Filesystem.get (conf);

        Path home = fs.gethomedirectory ()//Get user directory Path Hdfs_file = new Path (home, "hdfs_file");
    Upload (hdfs_file, new file ("E:\\code\\workspace_idea\\hadoopproject\\hadoop-hdfs-javaapi\\upload")); public void Upload (Path hdfs_file, File file) throws Exception {if (!fs.exists)) {fs.mkdirs (hdfs_file);
        } file[] files = file.listfiles ();
                if (Files!= null && files.length > 0) {for (file f:files) {//If the directory is created, if the file is uploaded
                    if (F.isdirectory ()) {Path Path = new Path (Hdfs_file, F.getname ());
                Upload (path, f);
                else {uploadfile (hdfs_file, F); }}} public void UploadFile (Path hdfs_file, file f) throws Exception {if (!fs.exi
        STS (Hdfs_file)) {fs.mkdirs (hdfs_file);
        } InputStream in = new FileInputStream (f);
        Fsdataoutputstream out = fs.create (new Path (Hdfs_file, F.getname ())); Ioutils.copybytes (in, out, 1024, true);
    True to automatically turn off resource logger.info ("file" + f.getname () + "upload complete"); }
}

3. File Bulk Download

Download all files in the HDFs/user/hadoop/hdfs_file directory to the download directory of the Project

Package Com.bigdata.hdfs;
Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.FSDataInputStream;
Import Org.apache.hadoop.fs.FileStatus;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.apache.hadoop.io.IOUtils;
Import Org.apache.log4j.Logger;

Import Org.junit.Test;
Import Java.io.File;
Import Java.io.FileOutputStream;

Import Java.io.OutputStream;
    public class Testdownloadhdfs {public static Logger Logger = Logger.getlogger (Testdownloadhdfs.class);
    public filesystem fs = null;

    public Configuration conf = null;

        @Test public void Testdownload () throws Exception {conf = new Configuration ();

        FS = Filesystem.get (conf);

        Path home = Fs.gethomedirectory (); Download (New Path (home.tostring () + "/hdfs_file"), New file ("e:\\code\\workspace_idea\\hadoopproject\\
    Hadoop-hdfs-javaapi\\download ")); } private void Download (Path home, File dir) throws Exception {if (!dir. Exists ()) {dir.mkdirs ();
        //Get all file information in the target directory in the Hadoop cluster filestatus[] filestatuses = Fs.liststatus (home); if (filestatuses!= null && filestatuses.length > 0) {//Loop traversal for (Filestatus f:filest  atuses) {//judgment is a directory or file, if the directory creates a recursive create if (F.isdirectory ()) {String name
                    = F.getpath (). GetName ();
                    File ToFile = new file (dir, name);
                Download (F.getpath (), tofile);
                else {///download file DownloadFile (F.getpath (), dir); '}} ' private void DownloadFile (path path, File dir) throws Exception {if (!dir.ex
        Ists ()) {dir.mkdirs ();
        File File = new file (dir, Path.getname ());
        Fsdatainputstream in = Fs.open (path);
        OutputStream out = new FileOutputStream (file);
  True: indicates that the stream is automatically closed when the operation completes      Ioutils.copybytes (in, out, 1024, true); }
}

Run Result: