1.people.txt
Soyo8, 35
Small week, 30
Xiao Hua, 19
soyo,88
2.
/**
* Created by Soyo on 17-10-10.
*Inference using reflection mechanismRDDMode
*/
Import Org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
Import Org.apache.spark.sql. {Encoder, sparksession}
Import Org.apache.spark.sql.SparkSession
Case class Person (name:String, Age:INT)
Object Rdd_to_dataframe {
ValSpark=sparksession.Builder (). Getorcreate ()
ImportSpark.implicits._//Support to put aRDDImplicitly converted to aDataFrame
DefMain (args:array[String]):Unit = {
Val df =Spark.sparkContext.textFile ("file:///home/soyo/Desktop/sparkProgramming test Data/people.txt ")
. Map (_.split (",")). Map (x=>Person (x (0), X (1). Trim.toint). TODF ()
Df.createorreplacetempview ("People")//CustomPeopleTable
Val persondf=Spark.sql ("SELECT * from people where age >=30")
Persondf.map (x=>"Name= "+x (0) +","+"Age= "+x (1). Show ()
//Convert to Table Form
Persondf.select (PERSONDF ("Name"). As ("Name"), PERSONDF ("Age"). As ("Age"). Show ()
Persondf. rdd.saveastextfile ("file:///home/soyo/ desktop /spark programming test Data / new data . txt")
}
}
Results:
+--------------+
| value|
+--------------+
| name =soyo8, age =35|
| Name = Small week, age =30|
| Name =soyo, age =88|
+--------------+
+-----+---+
| name | Age |
+-----+---+
|soyo8| 35|
| Small week | 30|
| soyo| 88|
+-----+---+
Spark SQL in RDD conversion to DataFrame