Upload two files to the input directory on HDFs
The code is as follows:
Import Java.io.ioexception;import Java.util.stringtokenizer;import Org.apache.hadoop.conf.configuration;import Org.apache.hadoop.fs.path;import Org.apache.hadoop.io.intwritable;import Org.apache.hadoop.io.text;import Org.apache.hadoop.mapreduce.job;import Org.apache.hadoop.mapreduce.mapper;import Org.apache.hadoop.mapreduce.reducer;import Org.apache.hadoop.mapreduce.lib.input.fileinputformat;import Org.apache.hadoop.mapreduce.lib.output.fileoutputformat;public class WordCount {public static class Tokenizermapper Extends Mapper<object, text, text, intwritable>{private final static intwritable one = new intwritable (1); Private text Word = new text (); public void Map (Object key, Text value, Context context) throws IOException, Interruptedexception { StringTokenizer ITR = new StringTokenizer (value.tostring ()); while (Itr.hasmoretokens ()) {Word.set (Itr.nexttoken ()); Context.write (Word, one); }}} public static CLass Intsumreducer extends reducer<text,intwritable,text,intwritable> {private intwritable result = new in Twritable (); public void reduce (Text key, iterable<intwritable> values, context context) throws Ioexc Eption, interruptedexception {int sum = 0; for (intwritable val:values) {sum + = Val.get (); } result.set (sum); Context.write (key, result); }} public static void Main (string[] args) throws Exception {Configuration conf = new configuration (); Job Job = job.getinstance (conf, "word count"); Job.setjarbyclass (Wordcount.class); Job.setmapperclass (Tokenizermapper.class); Job.setcombinerclass (Intsumreducer.class); Job.setreducerclass (Intsumreducer.class); Job.setoutputkeyclass (Text.class); Job.setoutputvalueclass (Intwritable.class); Fileinputformat.addinputpath (Job, New Path ("Hdfs://localhost:9000/input")); //The file path is specified directly here. Specified under Run configuration is also OK. But not configured Fileoutputformat.setoutputpath (Job, New Path ("HDFS://LOCALHOST:9000/OUTPUT1")); System.exit (Job.waitforcompletion (True)?0:1); } }
Note: There may be a log4j warning for the first execution of WordCount under Eclipse. The ability to create a file named Log4j.properties under SRC can eliminate the warning, such as the following:
Log4j.rootlogger=info,console log4j.appender.console= Org.apache.log4j.consoleappenderlog4j.appender.console.target=system.outlog4j.appender.console.threshold= debuglog4j.appender.console.layout= org.apache.log4j.patternlayoutlog4j.appender.console.layout.conversionpattern=[%d]%l%5p:%m%n Log4j.appender.debugfile=org.apache.log4j.rollingfileappenderlog4j.appender.debugfile.file=. /log/debugfile.log#log4j.appender.debugfile.file=debugfile.loglog4j.appender.debugfile.append= truelog4j.appender.debugfile.threshold=debuglog4j.appender.debugfile.layout= org.apache.log4j.patternlayoutlog4j.appender.debugfile.layout.conversionpattern=[%d]%l%5p:%m% nlog4j.appender.debugfile.maxfilesize=20mblog4j.appender.debugfile.maxbackupindex=10 log4j.logger.com.ibatis= Debuglog4j.logger.com.ibatis.common.jdbc.simpledatasource= Debuglog4j.logger.com.ibatis.common.jdbc.scriptrunner= Debuglog4j.logger.com.ibatis.sqlmap.engine.impl.sqlmapclientdelegate=debug log4j.logger.java.sql= DEBUGlog4j.logger.java.sql.Connection =INFOlog4j.logger.java.sql.Statement = DEBUGlog4j.logger.java.sql.PreparedStatement = DEBUGlog4j.logger.java.sql.ResultSet = DEBUG Log4j.logger.com.yuetao=debug
[Mapreduce]eclipse Write WordCount