Hadoop mapper&reducer debugging + test driver

Source: Internet
Author: User
Tags assert static class
Package test;
Import org.apache.hadoop.conf.Configured;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import org.apache.hadoop.io.IntWritable;
Import org.apache.hadoop.io.LongWritable;
Import Org.apache.hadoop.io.Text;
Import org.apache.hadoop.mapred.*;
Import Org.apache.hadoop.util.Tool;
Import Org.apache.hadoop.util.ToolRunner;
Import Org.junit.Assert;

Import Org.junit.Test;
Import java.io.IOException;
Import Java.net.URI;
Import java.util.ArrayList;
Import Java.util.Iterator;

Import java.util.List;
 /** * Created by the IntelliJ idea. * User:tony * DATE:11/20/11 * time:6:35 AM * To change this template use File | Settings |
 File Templates. */public class Temperaturemapreduce extends configured implements Tool {public static class Tempmapper extends Cebase implements Mapper<longwritable, Text, longwritable, text> {public void map (longwritable longwritable, Text text, outputcollector<longwritable, text> TextintwritableouTputcollector, Reporter Reporter) throws IOException {System.out.println (longwritable + "---" + text);
        Longwritable year = new Longwritable (long.valueof (text.tostring (). substring (0, 4));
        Text temperature = new text (text.tostring (). SUBSTRING (4, 6));
        Text result = text;
        SYSTEM.OUT.PRINTLN (year + "-+++++++++++++++++++-" + temperature);
    Textintwritableoutputcollector.collect (year, temperature); } public static class Tempreducer extends Mapreducebase implements Reducer<longwritable, Text, longwritable, Text&gt ; {public void reduce (longwritable key, iterator<text> values, outputcollector<longwritable, text> output,
        Reporter Reporter) throws IOException {int sum = 0;
        Text t = null;
        Long maxtemp = 0L;
            while (Values.hasnext ()) {Text tmp = Values.next ();
            System.out.println ("key:" + key + "it:" + tmp); Long temp = long.valueof (tmp.tostRing ());
                if (Temp > maxtemp) {maxtemp = temp;
            t = tmp;
    } output.collect (Key, New Text (maxtemp+ ""));
        @Test public void Testmapper () throws IOException {Tempmapper mymapper = new Tempmapper ();
        Text text = new text ("199951"); Outputcollector outputcollector = new outputcollector<longwritable,text> () {public void collect (Longwrit Able Resultkey, Text resultvalue) throws IOException {System.out.println ("Resultkey:" + Resultkey + "re
                Sultvalue: "+ resultvalue);
                Assert.asserttrue ("1999". Equals (Resultkey.tostring ()));
            Assert.asserttrue ("Wuyi". Equals (Resultvalue.tostring ());
        }
        };

    Mymapper.map (Null,text, outputcollector, NULL);
        @Test public void Testreducer () throws IOException {Tempreducer mymapper = new Tempreducer ();
 Text text = new text ("199951");       Longwritable longwritable = new Longwritable (1999);
        List List = new ArrayList ();
        List.add (New Text ("49"));
        List.add (New Text ("29"));



        List.add (New Text ("59")); Outputcollector outputcollector = new outputcollector<longwritable,text> () {public void collect (Longwrit Able Resultkey, Text resultvalue) throws IOException {System.out.println ("Resultkey:" + Resultkey + "re
                Sultvalue: "+ resultvalue);
                Assert.asserttrue ("1999". Equals (Resultkey.tostring ()));
            Assert.asserttrue ("resultvalue.tostring" () equals ());
        }
        };

    Mymapper.reduce (Longwritable, List.iterator (), outputcollector, NULL);
        public int run (string[] args) throws Exception {jobconf conf = new jobconf (getconf (), getclass ());
        Conf.setjobname ("WordCount");



        Conf.set ("Fs.default.name", "hdfs://192.168.126.133:9000/"); String output = "hdfs://192.168.126.133:9000/t1output ";
        FileSystem fs = Filesystem.get (uri.create (Output), conf);
        Path PATH = new path (output);
        if (fs.exists (path)) {Fs.delete (path, true);

        } fs.close ();
        Conf.setmapperclass (Tempmapper.class);
        Conf.setcombinerclass (Tempreducer.class);

        Conf.setreducerclass (Tempreducer.class);
        /*conf.setmapoutputkeyclass (Longwritable.class);
        Conf.setmapoutputvalueclass (Text.class);
        Conf.setoutputkeyclass (Text.class);
        Conf.setoutputvalueclass (Intwritable.class); * * Conf.setoutputkeyclass (Text.class);
        Conf.setoutputvalueclass (Intwritable.class); * * Conf.setinputformat (Textinputformat.class);
        Conf.setoutputformat (Textoutputformat.class); */fileinputformat.setinputpaths (conf, new Path (args[0));

        Fileoutputformat.setoutputpath (conf, new Path (args[1));
        Jobclient.runjob (conf);
    return 1; } public static VOID Main (string[] args) {try {string[] arg = new string[]{"Hdfs://192.168.126.133:9000/t1/temperature
            . txt "," hdfs://192.168.126.133:9000/t1output "};
        Toolrunner.run (New Temperaturemapreduce (), ARG);
        catch (Exception e) {e.printstacktrace ();
 }

    }


}

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.