Introduction to the Hadoop MapReduce Programming API series Statistics student score 1 (17)

Not much to say, directly on the code.

Code

Package zhouls.bigdata.myMapReduce.ScoreCount;

Import java.io.IOException;
Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.FSDataInputStream;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.mapreduce.InputSplit;
Import Org.apache.hadoop.mapreduce.JobContext;
Import Org.apache.hadoop.mapreduce.RecordReader;
Import Org.apache.hadoop.mapreduce.TaskAttemptContext;
Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Import Org.apache.hadoop.mapreduce.lib.input.FileSplit;
Import Org.apache.hadoop.util.LineReader;
/**
* Customize student performance reading and writing InputFormat
* Data Format reference: 19020090017 Xiao Ming 90 99 100 89 95
* @author Bertron
*/
public class Scoreinputformat extends fileinputformat< text,scorewritable >
{
@Override
Protected Boolean issplitable (Jobcontext context, Path filename)
{
TODO auto-generated Method Stub
return false;
}

@Override
Public recordreader< text,scorewritable > Createrecordreader (inputsplit inputsplit,
Taskattemptcontext context) throws IOException, Interruptedexception
{
return new Scorerecordreader ();
}
The two parameters in Recordreader fill in the type of key/value we expect to return, we expect key to be of type text, and value for scorewritable type encapsulates all student scores
public static class Scorerecordreader extends Recordreader<text, scorewritable >
{
Public Linereader in;//Line Reader
Public Text linekey;//Custom key type
Public scorewritable linevalue;//Custom value type
Public Text line;//per row data type

@Override
public void Close () throws IOException
{
if (in!=null)
{
In.close ();
}
}
@Override
Public Text Getcurrentkey () throws IOException, Interruptedexception
{
return linekey;
}
@Override
Public scorewritable GetCurrentValue () throws IOException,
Interruptedexception
{
return linevalue;
}
@Override
public float getprogress () throws IOException, Interruptedexception
{
return 0;
}
@Override
public void Initialize (Inputsplit input, Taskattemptcontext context)
Throws IOException, Interruptedexception
{
Filesplit split= (filesplit) input;
Configuration job=context.getconfiguration ();
Path File=split.getpath ();
FileSystem Fs=file.getfilesystem (Job);

Fsdatainputstream filein=fs.open (file);
In=new Linereader (filein,job);
Line=new Text ();
Linekey=new Text ();
Linevalue = new Scorewritable ();
}
This method reads each row of data, completing the custom key and value
@Override
public Boolean Nextkeyvalue () throws IOException, Interruptedexception
{
int Linesize=in.readline (line);//Data length per row
if (linesize==0) return false;
String[] pieces = line.tostring (). Split ("\\s+");//parse each row of data
if (pieces.length! = 7)
{
throw new IOException ("Invalid Record received");
}
Convert each student's grade to float type
float a,b,c,d,e;
try{
A = Float.parsefloat (Pieces[2].trim ());
b = Float.parsefloat (Pieces[3].trim ());
c = Float.parsefloat (Pieces[4].trim ());
D = float.parsefloat (Pieces[5].trim ());
E = Float.parsefloat (Pieces[6].trim ());
}catch (NumberFormatException NFE)
{
throw new IOException ("Error parsing floating poing value in record");
}
Linekey.set (pieces[0]+ "\ T" +pieces[1]);//Complete custom key data
Linevalue.set (A, B, C, D, e);//Encapsulate Custom Value data
return true;
}
}
}

Package zhouls.bigdata.myMapReduce.ScoreCount;

Import Java.io.DataInput;
Import Java.io.DataOutput;
Import java.io.IOException;
Import org.apache.hadoop.io.WritableComparable;
/**
* Learning achievement reading and writing class
* Data Format reference: 19020090017 Small Talk 90 99 100 89 95
* @author Bertron
*/
public class Scorewritable implements Writablecomparable<object >
{
Private float Chinese;
private float Math;
private float 中文版;
private float Physics;
private float chemistry;
Public scorewritable ()
{
}
Public scorewritable (float chinese,float math,float english,float physics,float chemistry)
{
This. Chinese = Chinese;
This. math = math;
This. 中文版 = 中文版;
This. Physics = Physics;
This. Chemistry = chemistry;
}
public void set (float chinese,float math,float english,float physics,float chemistry)
{
This. Chinese = Chinese;
This. math = math;
This. 中文版 = 中文版;
This. Physics = Physics;
This. Chemistry = chemistry;
}
public float Getchinese () {
return Chinese;
}
public float Getmath () {
return Math;
}
public float Getenglish () {
return 中文版;
}
public float getphysics ()
{
return Physics;
}
public float getchemistry ()
{
return chemistry;
}

public void ReadFields (Datainput in) throws IOException
{
Chinese = in.readfloat ();
Math = In.readfloat ();
中文版 = In.readfloat ();
Physics = In.readfloat ();
Chemistry = In.readfloat ();
}

public void Write (DataOutput out) throws IOException
{
Out.writefloat (Chinese);
Out.writefloat (Math);
Out.writefloat (中文版);
Out.writefloat (Physics);
Out.writefloat (chemistry);
}

public int compareTo (Object o)
{
return 0;
}
}

Package zhouls.bigdata.myMapReduce.ScoreCount;

Import java.io.IOException;
Import org.apache.hadoop.conf.Configuration;
Import org.apache.hadoop.conf.Configured;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.mapreduce.Job;
Import Org.apache.hadoop.mapreduce.Mapper;
Import Org.apache.hadoop.mapreduce.Reducer;
Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
Import Org.apache.hadoop.util.Tool;
Import Org.apache.hadoop.util.ToolRunner;
/**
* Student Score Statistics Hadoop program
* Data Format reference: 19020090017 Small Talk 90 99 100 89 95
* @author HUANGBQ
*/
public class Scorecount extends configured implements Tool
{
public static class Scoremapper extends mapper< text, scorewritable, text, scorewritable >
{
@Override
protected void Map (Text key, scorewritable value, Context context) throws IOException, Interruptedexception
{
Context.write (key, value);
}
}

public static class Scorereducer extends reducer< text, scorewritable, text, text >
{
Private text text = new text ();
protected void reduce (Text Key, iterable< scorewritable > Values, Context context)
Throws IOException, Interruptedexception
{
float totalscore=0.0f;
float Averagescore = 0.0f;
for (scorewritable ss:values)
{
Totalscore +=ss.getchinese () +ss.getmath () +ss.getenglish () +ss.getphysics () +ss.getchemistry ();
Averagescore +=TOTALSCORE/5;
}
Text.set (totalscore+ "\ t" +averagescore);
Context.write (Key, text);
}
}

public int run (string[] args) throws Exception
{
Configuration conf = new configuration ();//Read config file

Path MyPath = new Path (args[1]);
FileSystem HDFs = Mypath.getfilesystem (conf);//Create Output path
if (Hdfs.isdirectory (MyPath))
{
Hdfs.delete (MyPath, true);
}

Job Job = new Job (conf, "Scorecount");//New task
Job.setjarbyclass (Scorecount.class);//Set main class

Fileinputformat.addinputpath (Job, New Path (args[0]));//input path
Fileoutputformat.setoutputpath (Job, New Path (args[1]));//Output path

Job.setmapperclass (Scoremapper.class);//Mapper
Job.setreducerclass (Scorereducer.class);//Reducer

Job.setmapoutputkeyclass (Text.class);//Mapper key Output type
Job.setmapoutputvalueclass (Scorewritable.class);//Mapper value Output type

Job.setinputformatclass (Scoreinputformat.class);//Set Custom input formats

Job.waitforcompletion (TRUE);
return 0;
}

public static void Main (string[] args) throws Exception
{
string[] Args0 =
// {
"Hdfs://hadoopmaster:9000/score/score.txt",
"Hdfs://hadoopmaster:9000/out/score/"
// };

string[] Args0 =
{
"./data/score/score.txt",
"./out/score/"
};

int EC = Toolrunner.run (New Configuration (), New Scorecount (), ARGS0);
System.exit (EC);
}
}

Introduction to the Hadoop MapReduce Programming API series Statistics student score 1 (17)