Introduction to the Hadoop MapReduce Programming API series Statistics student score 1 (17)

Source: Internet
Author: User
Tags hadoop mapreduce

Not much to say, directly on the code.

Code

Package zhouls.bigdata.myMapReduce.ScoreCount;


Import java.io.IOException;
Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.FSDataInputStream;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.mapreduce.InputSplit;
Import Org.apache.hadoop.mapreduce.JobContext;
Import Org.apache.hadoop.mapreduce.RecordReader;
Import Org.apache.hadoop.mapreduce.TaskAttemptContext;
Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Import Org.apache.hadoop.mapreduce.lib.input.FileSplit;
Import Org.apache.hadoop.util.LineReader;
/**
* Customize student performance reading and writing InputFormat
* Data Format reference: 19020090017 Xiao Ming 90 99 100 89 95
* @author Bertron
*/
public class Scoreinputformat extends fileinputformat< text,scorewritable >
{
@Override
Protected Boolean issplitable (Jobcontext context, Path filename)
{
TODO auto-generated Method Stub
return false;
}

@Override
Public recordreader< text,scorewritable > Createrecordreader (inputsplit inputsplit,
Taskattemptcontext context) throws IOException, Interruptedexception
{
return new Scorerecordreader ();
}
The two parameters in Recordreader fill in the type of key/value we expect to return, we expect key to be of type text, and value for scorewritable type encapsulates all student scores
public static class Scorerecordreader extends Recordreader<text, scorewritable >
{
Public Linereader in;//Line Reader
Public Text linekey;//Custom key type
Public scorewritable linevalue;//Custom value type
Public Text line;//per row data type

@Override
public void Close () throws IOException
{
if (in!=null)
{
In.close ();
}
}
@Override
Public Text Getcurrentkey () throws IOException, Interruptedexception
{
return linekey;
}
@Override
Public scorewritable GetCurrentValue () throws IOException,
Interruptedexception
{
return linevalue;
}
@Override
public float getprogress () throws IOException, Interruptedexception
{
return 0;
}
@Override
public void Initialize (Inputsplit input, Taskattemptcontext context)
Throws IOException, Interruptedexception
{
Filesplit split= (filesplit) input;
Configuration job=context.getconfiguration ();
Path File=split.getpath ();
FileSystem Fs=file.getfilesystem (Job);

Fsdatainputstream filein=fs.open (file);
In=new Linereader (filein,job);
Line=new Text ();
Linekey=new Text ();
Linevalue = new Scorewritable ();
}
This method reads each row of data, completing the custom key and value
@Override
public Boolean Nextkeyvalue () throws IOException, Interruptedexception
{
int Linesize=in.readline (line);//Data length per row
if (linesize==0) return false;
String[] pieces = line.tostring (). Split ("\\s+");//parse each row of data
if (pieces.length! = 7)
{
throw new IOException ("Invalid Record received");
}
Convert each student's grade to float type
float a,b,c,d,e;
try{
A = Float.parsefloat (Pieces[2].trim ());
b = Float.parsefloat (Pieces[3].trim ());
c = Float.parsefloat (Pieces[4].trim ());
D = float.parsefloat (Pieces[5].trim ());
E = Float.parsefloat (Pieces[6].trim ());
}catch (NumberFormatException NFE)
{
throw new IOException ("Error parsing floating poing value in record");
}
Linekey.set (pieces[0]+ "\ T" +pieces[1]);//Complete custom key data
Linevalue.set (A, B, C, D, e);//Encapsulate Custom Value data
return true;
}
}
}

Package zhouls.bigdata.myMapReduce.ScoreCount;


Import Java.io.DataInput;
Import Java.io.DataOutput;
Import java.io.IOException;
Import org.apache.hadoop.io.WritableComparable;
/**
* Learning achievement reading and writing class
* Data Format reference: 19020090017 Small Talk 90 99 100 89 95
* @author Bertron
*/
public class Scorewritable implements Writablecomparable<object >
{
Private float Chinese;
private float Math;
private float 中文版;
private float Physics;
private float chemistry;
Public scorewritable ()
{
}
Public scorewritable (float chinese,float math,float english,float physics,float chemistry)
{
This. Chinese = Chinese;
This. math = math;
This. 中文版 = 中文版;
This. Physics = Physics;
This. Chemistry = chemistry;
}
public void set (float chinese,float math,float english,float physics,float chemistry)
{
This. Chinese = Chinese;
This. math = math;
This. 中文版 = 中文版;
This. Physics = Physics;
This. Chemistry = chemistry;
}
public float Getchinese () {
return Chinese;
}
public float Getmath () {
return Math;
}
public float Getenglish () {
return 中文版;
}
public float getphysics ()
{
return Physics;
}
public float getchemistry ()
{
return chemistry;
}

public void ReadFields (Datainput in) throws IOException
{
Chinese = in.readfloat ();
Math = In.readfloat ();
中文版 = In.readfloat ();
Physics = In.readfloat ();
Chemistry = In.readfloat ();
}

public void Write (DataOutput out) throws IOException
{
Out.writefloat (Chinese);
Out.writefloat (Math);
Out.writefloat (中文版);
Out.writefloat (Physics);
Out.writefloat (chemistry);
}

public int compareTo (Object o)
{
return 0;
}
}

Package zhouls.bigdata.myMapReduce.ScoreCount;


Import java.io.IOException;
Import org.apache.hadoop.conf.Configuration;
Import org.apache.hadoop.conf.Configured;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.mapreduce.Job;
Import Org.apache.hadoop.mapreduce.Mapper;
Import Org.apache.hadoop.mapreduce.Reducer;
Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
Import Org.apache.hadoop.util.Tool;
Import Org.apache.hadoop.util.ToolRunner;
/**
* Student Score Statistics Hadoop program
* Data Format reference: 19020090017 Small Talk 90 99 100 89 95
* @author HUANGBQ
*/
public class Scorecount extends configured implements Tool
{
public static class Scoremapper extends mapper< text, scorewritable, text, scorewritable >
{
@Override
protected void Map (Text key, scorewritable value, Context context) throws IOException, Interruptedexception
{
Context.write (key, value);
}
}

public static class Scorereducer extends reducer< text, scorewritable, text, text >
{
Private text text = new text ();
protected void reduce (Text Key, iterable< scorewritable > Values, Context context)
Throws IOException, Interruptedexception
{
float totalscore=0.0f;
float Averagescore = 0.0f;
for (scorewritable ss:values)
{
Totalscore +=ss.getchinese () +ss.getmath () +ss.getenglish () +ss.getphysics () +ss.getchemistry ();
Averagescore +=TOTALSCORE/5;
}
Text.set (totalscore+ "\ t" +averagescore);
Context.write (Key, text);
}
}

public int run (string[] args) throws Exception
{
Configuration conf = new configuration ();//Read config file

Path MyPath = new Path (args[1]);
FileSystem HDFs = Mypath.getfilesystem (conf);//Create Output path
if (Hdfs.isdirectory (MyPath))
{
Hdfs.delete (MyPath, true);
}

Job Job = new Job (conf, "Scorecount");//New task
Job.setjarbyclass (Scorecount.class);//Set main class

Fileinputformat.addinputpath (Job, New Path (args[0]));//input path
Fileoutputformat.setoutputpath (Job, New Path (args[1]));//Output path

Job.setmapperclass (Scoremapper.class);//Mapper
Job.setreducerclass (Scorereducer.class);//Reducer

Job.setmapoutputkeyclass (Text.class);//Mapper key Output type
Job.setmapoutputvalueclass (Scorewritable.class);//Mapper value Output type

Job.setinputformatclass (Scoreinputformat.class);//Set Custom input formats

Job.waitforcompletion (TRUE);
return 0;
}

public static void Main (string[] args) throws Exception
{
string[] Args0 =
// {
"Hdfs://hadoopmaster:9000/score/score.txt",
"Hdfs://hadoopmaster:9000/out/score/"
// };

string[] Args0 =
{
"./data/score/score.txt",
"./out/score/"
};

int EC = Toolrunner.run (New Configuration (), New Scorecount (), ARGS0);
System.exit (EC);
}
}

Introduction to the Hadoop MapReduce Programming API series Statistics student score 1 (17)

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.