userview--the second way (avoid set saturation in the first way), Java code implementation based on spark operator

userview--the second way (avoid set saturation in the first way), Java code implementation based on spark operator

test DataJava Code

1  PackageCom.hzf.spark.study;2 3 ImportJava.util.Map;4 ImportJava.util.Set;5 6 Importorg.apache.spark.SparkConf;7 ImportOrg.apache.spark.api.java.JavaPairRDD;8 ImportOrg.apache.spark.api.java.JavaRDD;9 ImportOrg.apache.spark.api.java.JavaSparkContext;Ten Importorg.apache.spark.api.java.function.Function; One Importorg.apache.spark.api.java.function.PairFunction; A ImportOrg.apache.spark.broadcast.Broadcast; -  - ImportScala. Tuple2; the  -  Public classUVANALYSIS02 { -      Public Static voidMain (string[] args) { -sparkconf conf =NewSparkconf (). Setappname ("Uv_ana"). Setmaster ("local") +. Set ("Spark.testing.memory", "2147480000"); -@SuppressWarnings ("Resource") +Javasparkcontext sc =Newjavasparkcontext (conf); Ajavardd<string> Logrdd = Sc.textfile ("UserLog1"); atString str = "View"; -         FinalBroadcast<string> broadcast =sc.broadcast (str); - UVANALYZEOPTZ (Logrdd, broadcast); -     } -      -     Private Static voidUVANALYZEOPTZ (javardd<string> Logrdd,FinalBroadcast<string>broadcast) { injavardd<string> Filteredlogrdd = Logrdd.filter (NewFunction<string, boolean>() { -            to             Private Static Final LongSerialversionuid = 1L; +  - @Override the              PublicBoolean Call (String v1)throwsException { *String Actionparam =Broadcast.value (); $String action = v1.split ("\ t") [5];Panax Notoginseng                 returnactionparam.equals (action); -             } the         }); +          Ajavapairrdd<string, string> Up2logrdd = Filteredlogrdd.maptopair (NewPairfunction<string, String, string>() { the  +             Private Static Final LongSerialversionuid = 1L; -  $ @Override $              PublicTuple2<string, String> call (String val)throwsException { -string[] splited = val.split ("\ t"); -String userId = splited[2]; theString pageId = splited[3]; -                  Wuyi                 return NewTuple2<string, string> (UserId + "_" + pageId,NULL); the             } -         }); Wu          -javapairrdd<string, iterable<string>> Groupup2logrdd =Up2logrdd.groupbykey (); About          $map<string, object> Countbykey = Groupup2logrdd.maptopair (NewPairfunction<tuple2<string,iterable<string>>, String, string>() { -  -             Private Static Final LongSerialversionuid = 1L; -  A @Override +              PublicTuple2<string, string> call (tuple2<string, iterable<string>> tuple)throwsException { theString PU =tuple._1; -string[] splited = Pu.split ("_"); $String pageId = splited[1]; the                 return NewTuple2<string, string> (PageId,NULL); the             } the }). Countbykey (); the          -Set<string> KeySet =Countbykey.keyset (); in          for(String key:keyset) { theSystem.out.println ("PAGEID:" +key+ "\tuv_count:" +Countbykey.get (key)); the         } About     } the}

View Coderesult

userview--the second way (avoid set saturation in the first way), Java code implementation based on spark operator