Irms simulation data generation and data analysis

First, data preparation
1. Generate random text every day, append 2 data to the text every hour, 100,000 bar each time
Random Data generation:
2,32 * * * * bash/mnt/jediael/irms/signalgenerator/signalgenerator.sh >>/home/jediael/sg.log 2>&1
Class: Signalgenerator

2. Import data text generated from the previous day into HDFs
0 * * * bash/mnt/jediael/irms/signalgenerator/copysignaltohdfs.sh >>/home/jediael/sg.log 2>&1


Second, data analysis
1. Perform a data analysis once a day to output the results to the HDFs text.
0 * * * bash/mnt/jediael/irms/signalparser/signalparser.sh >>/home/jediael/sg.log 2>&1
Class: Signalparser

Program Files:

/mnt/jediael/irms/signalgenerator/signalgenerator.sh

/mnt/jediael/irms/signalgenerator/copysignaltohdfs.sh

#!/bin/bashexport java_home=/usr/java/jdk1.7.0_51path=/mnt/jediael/hadoop-1.2.1/bin/:/mnt/jediael/hbase-0.94.26 /bin:/mnt/jediael/tomcat-7.0.54/bin: $JAVA _home/bin: $PATHhadoop fs-copyfromlocal/mnt/jediael/irms/ signalgenerator/' date-d '-1 day "+%y%m%d". Txt/irms/signal >> Sg.log

/mnt/jediael/irms/signalparser/signalparser.sh

#!/bin/bashexport java_home=/usr/java/jdk1.7.0_51path=/mnt/jediael/hadoop-1.2.1/bin/:/mnt/jediael/hbase-0.94.26 /bin:/mnt/jediael/tomcat-7.0.54/bin: $JAVA _home/bin: $PATHhadoop jar signalgenerator.jar  /irms/signal/' date-d ' -1 Day "+%y%m%d '. txt  /irms/result/' date-d"-1 day "+%y%m%d"

Signalgenerator.java

Package Com.gmcc.irms.util;import Java.io.fileoutputstream;import Java.io.ioexception;import java.io.OutputStream; Import Java.text.simpledateformat;import java.util.random;//This class is used to simulate the generation of signaling data, generating 100,000 public class signalgenerator each time {/ /business type, such as inbound, outbound, texting, SMS, Internet, WLAN, etc. private int activetype = 0;private string Getnextsign () {String sign = ""; Random rand = new Random (); activetype = Rand.nextint (6);//caller number string callnum = "1390222" + rand.nextint (9) + Rand.nextint (9) + rand.nextint (9) + rand.nextint (9);//called number string becallnum = "1390222" + rand.nextint (9) + rand.nextint (9) + Rand.nexti NT (9) + rand.nextint (9);//Duration, or traffic string callduration = ""; for (int i = 0; I < 16; i++) {callduration + = Rand.nextint (9);} Sign = Activetype + callnum + becallnum + callduration;for (int i = 0; i <, i++) {sign + = Rand.nextint (9);} return sign;} public static void Main (string[] args) throws IOException {SimpleDateFormat SDF = new SimpleDateFormat ("YyyyMMdd"); Long Ti me = System.currenttimemillis (); String FileName= Sdf.format (time) + ". txt"; outputstream os = new FileOutputStream (FileName, true); Signalgenerator sg = new Signalgenerator (); String newline = system.getproperty ("Line.separator"); for (int i = 0; i < 100000; i++) {Os.write ((sg.getnextsign () + ne) Wline). GetBytes ());} Os.flush (); Os.close ();}}

Signalparser.java

Package Com.gmcc.irms.signal;import Java.io.ioexception;import Org.apache.hadoop.fs.path;import Org.apache.hadoop.io.longwritable;import Org.apache.hadoop.io.text;import Org.apache.hadoop.mapreduce.job;import Org.apache.hadoop.mapreduce.mapper;import Org.apache.hadoop.mapreduce.reducer;import Org.apache.hadoop.mapreduce.lib.input.fileinputformat;import Org.apache.hadoop.mapreduce.lib.output.fileoutputformat;public class Signalparser {public static void main (string[] args) throws Exception {Job job = new Job (); Job.setjarbyclass (Signalparser.class); Job.setjobname ("Signal parser"); Fileinputformat.addinputpath (Job, New Path (Args[0])); Fileoutputformat.setoutputpath (Job, New Path (args[1)); Job.setmapperclass (Signalparsermapper.class); Job.setreducerclass (Signalparserreducer.class); Job.setoutputkeyclass (Text.class); Job.setOutputValueClass ( Text.class); System.exit (Job.waitforcompletion (true)? 0:1);}} Class Signalparsermapper extends Mapper<longwritable, text, text, text> {@Overridepublic void map (longwritable key, Text value, Context context) throws IOException, interruptedexception {String line = Value.tost Ring ();//Business type String activetype = line.substring (0, 1);//caller number String customer = line.substring (1, 12);//length of call, Web Duration, Wla n Time int duration = Integer.parseint (line.substring (+)), Context.write (new text (customer), new text (Activetype + "," + Duration));}} Class Signalparserreducer extends Reducer<text, text, text, text> {@Overridepublic void reduce (text key, ITERABLE&L T Text> values, Context context) throws IOException, Interruptedexception {//Total call duration int sumcallduration = 0;//calls int  Calltimes = 0;//send sms number int smstimes = 0;//Internet total time int sumwebduration = 0;//number of internet times int webtimes = 0;//wlan total time int sumwlanduration = 0;//wlan times int wlantimes = 0; string[] ValueArray = null;int Activetype = -1;int Duration = -1;for (Text value:values) {ValueArray = Value.tostring (). SPL It (","); System.out.println (valuearray[0]+ "a" +valuearray[1]) Activetype = Integer.parseint (valuearray[0]);d UratIon = Integer.parseint (valuearray[1]); if (Activetype = = 0) {//outbound sumcallduration + = duration;calltimes++;} else if (Activetype = = 2) {//hair smssmstimes++;} else if (Activetype = = 4) {//internet sumwebduration + = Duration;webtimes + +;} else if (Activetype = = 5) {//wlansumwlanduration + = Duration;wlantimes + +;} Else{}}context.write (Key, New Text (sumcallduration + "\ T" + calltimes + "\ t" +smstimes+ "\ T" + sumwebduration+ "\ T" + Webtim es+ "\ T" + sumwebduration+ "\ T" + Webtimes));}}

Irms simulation data generation and data analysis