File read/Write tool class on HDFs Demo

read file on HDFs demo

Package com.utils;
Import java.io.IOException;

Import java.util.List;
Import Org.apache.hadoop.fs.FSDataInputStream;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.util.LineReader;
Import Org.slf4j.Logger;

Import Org.slf4j.LoggerFactory;
	 /** * Read files on HDFs * * @author chichuduxing * @date September 19, 2016 PM 14:19:14 */public class Hdfsreader {/** * Log Object

	*/protected static final Logger Logger = Loggerfactory.getlogger (Hdfsreader.class);

	Private FileSystem _fs = null;

	/** * Line Reader */private linereader _linereader = null;

	/** * HDFS Input stream */private fsdatainputstream _fsinputstream = null;
	/** * Constructor * * @param FS * */Public Hdfsreader (FileSystem fs) {THIS._FS = FS; }/** * Initialize * * @param file * path to read (/tmp/readdemo.txt) * @return */public boolean Init (Strin G file) {if (null = = File | | file.isempty ()) {logger.error ("file name is null");
		return false;

			} try {Path file_path = new Path (file);
				if (!_fs.exists (File_path)) {logger.error (file + "not exist!");
			return false;

			}//Open data stream this._fsinputstream = This._fs.open (File_path);

			This._linereader = new Linereader (_fsinputstream, _fs.getconf ());
		return true;
			} catch (Exception e) {logger.error ("Create line reader failed-" + e.getmessage (), E);
		return false; }}/** * Start reading data by line from File * * @param dataList * Read the file information * @param linecont * Number of lines per read * Whether the @return read successfully */public boolean next (list<string> dataList, int linecont) {if (null = = This._linereader | | nu
		ll = = DataList) {return false;
		} Text line = new text (); while (Datalist.size () < Linecont) {try {///file already read if (This._linereader.readline (line) <= 0) {Clo
					Sefilestream ();
				Break
				}} catch (Exception e) {logger.error ("read file failed--" + e.getmessage (), E); ClosefilesTream ();
			return false;
		} datalist.add (Line.tostring ());

		} logger.info ("Get Data count:" + datalist.size ());
	return true; }/** * Close file stream and row reader */private void Closefilestream () {try {if (This._fsinputstream! = NULL | | this._lineread
				Er! = null) {_fsinputstream.close ();
			_linereader.close ();
		}} catch (IOException e) {logger.error ("Closefilestream () failed--" + e.getmessage (), E);
			} finally {_fsinputstream = null;
		_linereader = null;
 }
	}
}

write files to HDFs on demo

Package com.utils;
Import java.io.IOException;

Import Java.io.OutputStream;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.slf4j.Logger;

Import Org.slf4j.LoggerFactory;
	 /** * Write file to HDFs * * @author chichuduxing * @date September 19, 2016 morning 11:19:14 */public class Hdfswriter {/** * Log Object

	*/private static final Logger Logger = Loggerfactory.getlogger (Hdfswriter.class);

	/** * HDFs Upload directory */private final String _hdfsoutdirectory;

	/** * Temp File * */private Path _tmpfilepath;

	/** * HDFs File handle */private FileSystem _FS = null;

	/** * Write file stream handle */private outputstream _outputstream = null;

	/** * Whether there is write data */private Boolean _ifwritedata = false; /** * @param FS * @param outpath * Output file path (/tmp/output/test.txt) */public Hdfswriter (FileSystem FS, St
		Ring Outpath) {this._hdfsoutdirectory = Outpath;
	THIS._FS = FS; }/** * Initialize HDFs write directory, handle * * @return */public boolean init () throws Exception {
		Logger.info ("Hdfswriter.init () Start ...");
		try {//clean up temporary data initoutstream ();

			} catch (Exception e) {logger.error ("Hdfsloader.init () failed.", e);
		return false;
		} logger.info ("Hdfswriter.init () done ...");
	return true; }/** * Initialize input stream * * @return */private void Initoutstream () throws Exception {try {//Generate temporary File String TM
			P_file_name = This._hdfsoutdirectory + ". tmp";

			This._tmpfilepath = new Path (tmp_file_name);
			if (this._fs.exists (This._tmpfilepath)) {This._fs.delete (This._tmpfilepath, true); } if (!this._fs.createnewfile (This._tmpfilepath)) {throw new Exception ("Create TMP HDFs file failed.
			--"+ This._tmpfilepath);

			} logger.info ("Create HDFs tmp File success:" + Tmp_file_name);
		Gets the file stream handle This._outputstream = This._fs.append (This._tmpfilepath);
			} catch (Exception e) {this._outputstream = null;
		throw new Exception ("Hdfswriter.initoutstream () failed.", e); }}/** * Write data to HDFs * * @param Data */public void WriteData (String data) {if (null = = Data | | data.isempty () | | null = = this._outputstream) re

		Turn

		String to bytes byte[] bcp_bytes = data.tostring (). GetBytes ();
				Write bcp stream data to HDFs if (0 < bcp_bytes.length) {try {this._outputstream.write (bcp_bytes);

				This._outputstream.flush (); Logger.info ("Upload bcp data success.

				--"+ this._tmpfilepath.tostring ());
			This._ifwritedata = true;

				} catch (Exception ex) {Logger.error ("# #hdfs Write error##:" + ex.getmessage (), ex);
				Transfer temporary files closefilestream ();
				try {//Reset write stream Initoutstream ();
				} catch (Exception e) {logger.error (E.getmessage ()); }}}}/** * Close file Write stream */public void Closefilestream () {try {//Close file stream if (null! = This._outputstrea
				m) {Logger.info ("Close the HDFs file stream.");
				This._outputstream.close ();
			This._outputstream = null; }} catch (Exception e) {logger.error ("Close the HDFs file StreaM Failed,file is: "+ This._tmpfilepath, E);
		} finally {this._outputstream = null;
	}//Transfer temporary file RenameFile ();

		}/** * Rename file */private void RenameFile () {//whether there is write data to prevent empty file if (!this._ifwritedata) return;

		Rename file name path Res_path = new Path (this._hdfsoutdirectory);
			try {if (this._fs.exists (Res_path)) {This._fs.delete (Res_path, true); } if (!this._fs.rename (This._tmpfilepath, Res_path)) {logger.error ("rename [" + This._tmpfilepath + "] to [" + Res
				_path + "] failed.");
			Return
		}} catch (IOException e) {logger.error ("rename the HDFs file Failed,file is:" + This._tmpfilepath, E);
 } logger.info ("rename [" + This._tmpfilepath + "] to [" + Res_path + "] ok.");}}