Using Eclipse to develop Hadoop under windows

I. Configuring the Hadoop plug-in

1. Installing plugins

Copy the Hadoop-eclipse-plugin-1.1.2.jar to the Eclipse/plugins directory and restart eclipse

2. Open the MapReduce view

Window---open perspective, other select Map/reduce icon is a blue elephant

3. Add a MapReduce environment

At the lower end of eclipse, there will be a tab "Map/reduce Locations" next to the console, right-click in the space below and select "New Hadoop location", configured as follows:

Location name (from a name)

map/reduce Master (the IP and port of the Job Tracker, based on the mapred.job.tracker of the Mapred-site.xml configuration )

DFS Master (Name node IP and port, based on fs.default.name configured in Core-site.xml)

4. Use Eclipse to modify HDFS content

After the previous step, the left "Project Explorer" will appear in the configuration of HDFs, right-click, you can create new files, delete files and other operations.

Note: The changes cannot be displayed immediately after each operation in eclipse and must be refreshed.

Two. Developing Hadoop programs

1. Wordcount.java

public class WordCount {public static class Tokenizermapper Extendsmapper<object, text, text, intwritable> {private Final static intwritable one = new intwritable (1);p rivate Text word = new text ();p ublic void map (Object key, text value, Context context) throws IOException, interruptedexception {stringtokenizer ITR = new StringTokenizer (value.tostring ()); while (Itr.hasmoretokens ()) {Word.set (Itr.nexttoken ()); Context.write (Word, one);}}} public static class Intsumreducer Extendsreducer<text, Intwritable, Text, intwritable> {private intwritable result = new Intwritable ();p ublic void reduce (Text key, iterable<intwritable> Values,context Context) throws IOException, interruptedexception {int sum = 0;for (intwritable val:values) {sum + = Val.get ();} Result.set (sum); Context.write (key, result); }}public static void Main (string[] args) throws Exception {//Use program to generate temporary jar files file Jarfile = Ejob.createtempjar ("bin"); Ejob.addclasspath ("/cloud/hadoop/conf"); ClassLoader ClassLoader = Ejob. getClassLoader (); Thread.CurrentThread (). Setcontextclassloader (ClassLoader);//Set Hadoop configuration parameters Configuration conf = new configuration ();  Conf.set ("Fs.default.name", "hdfs://hadoop001:9000");  Conf.set ("Hadoop.job.user", "root"); Conf.set ("Mapred.job.tracker", "hadoop001:9001"); Job Job = new Job (conf, "word count");((jobconf) job.getconfiguration ()). Setjar (Jarfile.tostring ()); Job.setjarbyclass (Wordcount.class); Job.setmapperclass (Tokenizermapper.class); Job.setcombinerclass (Intsumreducer.class); Job.setreducerclass (Intsumreducer.class); Job.setoutputkeyclass (Text.class); Job.setoutputvalueclass ( Intwritable.class); String input = "Hdfs://hadoop001:9000/user/root/tmp_file_1"; String output = "hdfs://hadoop001:9000/user/root/tmp_file_2"; Fileinputformat.addinputpath (Job, new Path (input)); Fileoutputformat.setoutputpath (Job, new Path (output)); System.exit (Job.waitforcompletion (true)? 0:1);}}

2. Ejob.java

Generate temporary JAR file class public class Ejob {//To declare global fieldprivate static list<url> ClassPath = new Arraylist<url  > ();/To declare methodpublic static file Createtempjar (String root) throws IOException {if (!new File (root). Exists ()) {return null;} Manifest Manifest = new Manifest (), Manifest.getmainattributes (). Putvalue ("Manifest-version", "1.0"); final File Jarfile = File.createtempfile ("ejob-", ". Jar", New File (System.getproperty ("Java.io.tmpdir")); Runtime.getruntime (). Addshutdownhook (New Thread () {public void run () {jarfile.delete ();}); Jaroutputstream out = new Jaroutputstream (new FileOutputStream (jarfile), manifest); Createtempjarinner (out, New File ( Root), ""); Out.flush (); Out.close (); return jarfile;} private static void Createtempjarinner (Jaroutputstream out, File f,string base) throws IOException {if (F.isdirectory ()) { file[] fl = F.listfiles (); if (Base.length () > 0) {base = base + "/";} for (int i = 0; i < fl.length; i++) {Createtempjarinner (out, fl[i], base + fl[i). GetName ());}} else {out.putnextentry, new jarentry (base); FileInputStream in = new FileInputStream (f); byte[] buffer = new Byte[1024];int n = in.read (buffer), while (n! =-1) {OUT.WR ITE (buffer, 0, n); n = in.read (buffer);} In.close ();}} public static ClassLoader getClassLoader () {ClassLoader parent = Thread.CurrentThread (). Getcontextclassloader (); if ( parent = = NULL) {parent = EJob.class.getClassLoader ();} if (parent = = NULL) {parent = Classloader.getsystemclassloader ();} return new URLClassLoader (Classpath.toarray (new url[0]), parent); public static void Addclasspath (String component) {if (component! = null) && (component.length () > 0)) {try {F] Ile f = new File (component), if (F.exists ()) {URL key = F.getcanonicalfile (). Tourl (); if (!classpath.contains (key)) {CLASSP Ath.add (key);}}} catch (IOException e) {}}}}

Using Eclipse to develop Hadoop under windows