Use Java API to access HDFS files

1. Repeat the configuration file core-site.xml

To use the Java client to access the files on HDFS, I have to say that the configuration file hadoop-0.20.2/conf/core-site.xml, at first I am here to eat a big loss, so I can not connect to HDFS, files cannot be created or read.

<? Xml version = "1.0"?>
<? Xml-stylesheet type = "text/xsl" href = "configuration. xsl"?>

<Configuration>
<! --- Global properties -->
<Property>
<Name> hadoop. tmp. dir </name>
<Value>/home/zhangzk/hadoop </value>
<Description> A base for other temporary directories. </description>
</Property>
<! -- File system properties -->
<Property>
<Name> fs. default. name </name>
<Value> hdfs: // linux-zzk-113: 9000 </value>
</Property>
</Configuration>

Configuration item: hadoop. tmp. dir indicates the directory location where metadata is stored on the named node. For data nodes, the directory where file data is stored on the node.

Configuration item: fs. default. name indicates the IP address and port number. The default value is file: //. For Java APIs, the configured URL address must be used to connect to HDFS. For data nodes, the data node uses this URL to access the named node.

 

2. Use Java API to access HDFS files and directories

Package com. demo. hdfs;

Import java. io. BufferedInputStream;
Import java. io. FileInputStream;
Import java. io. FileNotFoundException;
Import java. io. FileOutputStream;
Import java. io. IOException;
Import java. io. InputStream;
Import java. io. OutputStream;
Import java.net. URI;

Import org. apache. hadoop. conf. Configuration;
Import org. apache. hadoop. fs. FSDataInputStream;
Import org. apache. hadoop. fs. FSDataOutputStream;
Import org. apache. hadoop. fs. FileStatus;
Import org. apache. hadoop. fs. FileSystem;
Import org. apache. hadoop. fs. Path;
Import org. apache. hadoop. io. IOUtils;
Import org. Apache. hadoop. util. progressable;

/**
* @ Author zhangzk
*
*/
Public class filecopytohdfs {

Public static void main (string [] ARGs) throws exception {
Try {
// Uploadtohdfs ();
// Deletefromhdfs ();
// Getdirectoryfromhdfs ();
Appendtohdfs ();
Readfromhdfs ();
} Catch (exception e ){
// Todo auto-generated Catch Block
E. printstacktrace ();
}
Finally
{
System. Out. println ("success ");
}
}

 

/** Upload files to HDFS */

Private Static void uploadtohdfs () throws filenotfoundexception, ioexception {
String localsrc = "D: // qq.txt ";
String DST = "HDFS: // 192.168.0.113: 9000/user/zhangzk/qq.txt ";
Inputstream in = new bufferedinputstream (New fileinputstream (localsrc ));
Configuration conf = new configuration ();

Filesystem FS = filesystem. Get (URI. Create (DST), conf );
Outputstream out = FS. Create (New Path (DST), new progressable (){
Public void progress (){
System. Out. Print (".");
}
});
Ioutils. copybytes (In, out, 4096, true );
}
 

/** Read files from HDFS */
Private Static void readfromhdfs () throws filenotfoundexception, ioexception {
String DST = "HDFS: // 192.168.0.113: 9000/user/zhangzk/qq.txt ";
Configuration conf = new configuration ();
Filesystem FS = filesystem. Get (URI. Create (DST), conf );
Fsdatainputstream hdfsinstream = FS. Open (New Path (DST ));

Outputstream out = new fileoutputstream ("D:/qq-hdfs.txt ");
Byte [] iobuffer = new byte [1024];
Int readlen = hdfsinstream. Read (iobuffer );

While (-1! = Readlen ){
Out. Write (iobuffer, 0, readlen );
Readlen = hdfsinstream. Read (iobuffer );
}
Out. Close ();
Hdfsinstream. Close ();
FS. Close ();
}
 

/** Add content in append mode to the end of the file on HDFS; Note: file update, you need to add <property> <Name> DFS in the hdfs-site.xml. append. support </Name> <value> true </value> </property> */
Private Static void appendtohdfs () throws filenotfoundexception, ioexception {
String DST = "HDFS: // 192.168.0.113: 9000/user/zhangzk/qq.txt ";
Configuration conf = new configuration ();
Filesystem FS = filesystem. Get (URI. Create (DST), conf );
Fsdataoutputstream out = FS. append (New Path (DST ));

Int readlen = "zhangzk add by HDFS Java API". getbytes (). length;

While (-1! = Readlen ){
Out. write ("zhangzk add by hdfs java api". getBytes (), 0, readLen );
}
Out. close ();
Fs. close ();
}
 

/** Delete an object from HDFS */
Private static void deleteFromHdfs () throws FileNotFoundException, IOException {
String dst = "hdfs: // 192.168.0.113: 9000/user/zhangzk/qq-bak.txt ";
Configuration conf = new Configuration ();
FileSystem fs = FileSystem. get (URI. create (dst), conf );
Fs. deleteOnExit (new Path (dst ));
Fs. close ();
}
 

/** Traverse the files and directories on HDFS */
Private static void getDirectoryFromHdfs () throws FileNotFoundException, IOException {
String DST = "HDFS: // 192.168.0.113: 9000/user/zhangzk ";
Configuration conf = new configuration ();
Filesystem FS = filesystem. Get (URI. Create (DST), conf );
Filestatus filelist [] = FS. liststatus (New Path (DST ));
Int size = filelist. length;
For (INT I = 0; I <size; I ++ ){
System. out. println ("name:" + filelist [I]. getpath (). getname () + "/T/tsize:" + filelist [I]. getlen ());
}
FS. Close ();
}

}

 

Note: append operations are not supported since the hadoop-0.21 version. For append operations, see a document on javaeye.

Http://dongyajun.javaeye.com/blog/643391