Object Logichandle {def main (args:array[string]) {//Add this will not report execution errorVal Path =NewFile ("."). Getcanonicalpath () system.getproperties (). Put ("Hadoop.home.dir", path); NewFile ("./bin")). Mkdirs (); NewFile ("./bin/winutils.exe")). CreateNewFile (); //val sparkconf = new sparkconf (). Setappname ("Sensorrealtime"). Setmaster ("local[2]")Val sparkconf =NewSparkconf (). Setappname ("Sensorrealtime") Val SSC=NewStreamingContext (sparkconf, Seconds (20)) Val hostname= "localhost"Val Port= 2345Val Storagelevel=storagelevel.memory_only Val Flumestream=Flumeutils.createstream (SSC, hostname, port, Storagelevel) Val LHC=NewLogichandleclass (); //Log Formatting TemplatesVal sdf =NewSimpleDateFormat ("Yyyy-mm-dd HH:mm:ss"); Val Sdfhour=NewSimpleDateFormat ("HH"); Val Sdfminute=NewSimpleDateFormat ("MM") //The hash object that stores the data key/value stored according to the rules of the document, using the key/value of each statistic indicatorvar redismap =Newhashmap[string, String]
Flumestream.foreachrdd (Rdd={Val Events=Rdd.collect ()//println ("Event count:" + events.length)var i = 1 for(Event <-events) {val Sensorinfo=NewString (Event.event.getBody.array ())//single-Line Records//single-line record formattingVal arrayfileds = Sensorinfo.split (",") if(Arrayfileds.length = = 6) {val shopid= Arrayfileds (0)//in-store numberVal Floorid= shopid.substring (0, 5)//Floor NumberVal mac = arrayfileds (1) Val ts= Arrayfileds (2). Tolong//time StampVal time = Sdf.format (TS * 1000) var hour= Sdfhour.format (TS * 1000) var minute= Sdfminute.format (TS * 1000) var allminute= Hour.toint * 60 +minute.toint val x= Arrayfileds (3) Val y= Arrayfileds (4) Val level= Arrayfileds (5) //Behind is my business code, omitted } } //stored in RedisThe LHC. SetAll (Redismap)}) Ssc.start () Ssc.awaittermination ()}}
Share the Scala code that spark streaming integrates with Flume.