Hadoop sample Program WordCount detailed and examples

1. Diagram of MapReduce

2. Resume Process:

Input:

Hello World Bye

Hello Hadoop Bye Hadoop

Bye Hadoop Hello Hadoop

MAP:

<Hello,1>

<World,1>

<Bye,1>

<World,1>

<Hello,1>

<Hadoop,1>

<Bye,1>

<Hadoop,1>

<Bye,1>

<Hadoop,1>

<Hello,1>

<Hadoop,1>

Sort:

<Bye,1>

<Bye,1>

<Bye,1>

<Hadoop,1>

<Hadoop,1>

<Hadoop,1>

<Hadoop,1>

<Hello,1>

<Hello,1>

<Hello,1>

<World,1>

<World,1>

Combine:

<Bye,1,1,1>

<Hadoop,1,1,1,1>

<Hello,1,1,1>

<World,1,1>

Reduce:

<Bye,3>

<Hadoop,4>

<Hello,3>

<World,2>

3. code example:

[C-sharp]View Plaincopy

2. Package Com.felix;
4. Import java.io.IOException;
6. Import Java.util.Iterator;
8. Import Java.util.StringTokenizer;
10. Import Org.apache.hadoop.fs.Path;
12. Import org.apache.hadoop.io.IntWritable;
14. Import org.apache.hadoop.io.LongWritable;
16. Import Org.apache.hadoop.io.Text;
18. Import Org.apache.hadoop.mapred.FileInputFormat;
20. Import Org.apache.hadoop.mapred.FileOutputFormat;
22. Import org.apache.hadoop.mapred.JobClient;
24. Import org.apache.hadoop.mapred.JobConf;
26. Import Org.apache.hadoop.mapred.MapReduceBase;
28. Import Org.apache.hadoop.mapred.Mapper;
30. Import Org.apache.hadoop.mapred.OutputCollector;
32. Import Org.apache.hadoop.mapred.Reducer;
34. Import Org.apache.hadoop.mapred.Reporter;
36. Import Org.apache.hadoop.mapred.TextInputFormat;
38. Import Org.apache.hadoop.mapred.TextOutputFormat;
40. /**
42. *
44. * Description: WordCount explains by Felix
46. * @author Hadoop Dev Group
48. */
50. public class WordCount
52. {
54. /**
56. * Mapreducebase class: Implements the base class for Mapper and Reducer interfaces (where the method simply implements the interface without doing anything)
58. * Mapper Interface:
60. * Writablecomparable Interface: Classes that implement writablecomparable can be compared to each other. All classes that are used as keys should implement this interface.
62. * Reporter can be used to report the running progress of the entire application, which is not used in this example.
64. *
66. */
68. public static class Map extends Mapreducebase implements
70. mapper<longwritable, text, text, intwritable>
72. {
74. /**
76. * longwritable, Intwritable, Text are classes implemented in Hadoop to encapsulate Java data types that implement the Writablecomparable interface.
78. * Can be serialized to facilitate data exchange in a distributed environment, and you can consider them as long,int,string alternatives.
80. */
82. Private final static intwritable one = new intwritable (1);
84. Private text Word = new text ();
88. /**
90. * The map method in the Mapper interface:
92. * void Map (K1 key, V1 value, outputcollector<k2,v2> output, Reporter Reporter)
94. * Map a single input k/v pair to an intermediate k/v
96. * The output pair is not required and the input pair is the same type, the input pair can be mapped to 0 or more output pairs.
98. * Outputcollector Interface: Collects <k,v> pairs of mapper and reducer outputs.
100. * Outputcollector interface Collect (k, V) method: Add a (k,v) to the output
102. */
104. public void Map (longwritable key, Text value,
106. Outputcollector<text, intwritable> output, Reporter Reporter)
108. Throws IOException
110. {
112. String line = value.tostring ();
114. StringTokenizer tokenizer = new StringTokenizer (line);
116. while (Tokenizer.hasmoretokens ())
118. {
120. Word.set (Tokenizer.nexttoken ());
122. Output.collect (Word, one);
124. }
126. }
128. }
130. public static class Reduce extends Mapreducebase implements
132. Reducer<text, Intwritable, Text, intwritable>
134. {
136. public void reduce (Text key, iterator<intwritable> values,
138. Outputcollector<text, intwritable> output, Reporter Reporter)
140. Throws IOException
142. {
144. int sum = 0;
146. while (Values.hasnext ())
148. {
150. Sum + = Values.next (). get ();
152. }
154. Output.collect (Key, New intwritable (sum));
156. }
158. }
160. public static void Main (string[] args) throws Exception
162. {
164. /**
166. * Jobconf:map/reduce job Configuration class, describing the work performed by Map-reduce to the Hadoop framework
168. * Construction Method: jobconf (), jobconf (Class exampleclass), jobconf (Configuration conf), etc.
170. */
172. jobconf conf = new jobconf (wordcount.class);
174. Conf.setjobname ("WordCount"); Set a user-defined job name
176. Conf.setoutputkeyclass (Text.class); Set the key class for the job's output data
178. Conf.setoutputvalueclass (Intwritable.class); Set the value class for the job output
180. Conf.setmapperclass (Map.class); Set the Mapper class for the job
182. Conf.setcombinerclass (Reduce.class); Set the Combiner class for the job
184. Conf.setreducerclass (Reduce.class); To set the reduce class for a job
186. Conf.setinputformat (Textinputformat.class); To set the InputFormat implementation class for a map-reduce task
188. Conf.setoutputformat (Textoutputformat.class); To set the OutputFormat implementation class for a map-reduce task
190. /**
192. * InputFormat describes the input definition of the job in Map-reduce
194. * Setinputpaths (): Sets the path array as an input list for the Map-reduce job
196. * Setinputpath (): Sets the path array as the output list for the Map-reduce job
198. */
200. Fileinputformat.setinputpaths (conf, new Path (Args[0]));
202. Fileoutputformat.setoutputpath (conf, new Path (Args[1]));
204. Jobclient.runjob (conf); Run a job
206. }
208. }

Hadoop sample program WordCount and examples (RPM)