RDD & java 類 （反射）構建 DataFrame ---java code

import java.util.List;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.api.java.function.Function;import org.apache.spark.sql.DataFrame;import org.apache.spark.sql.Row;import org.apache.spark.sql.SQLContext;/** * 使用反射的方式將RDD轉換成為DataFrame * 1、自訂的類必須是public * 2、自訂的類必須是可序列化的 * 3、RDD轉成DataFrame的時候，他會根據自訂類中的欄位名進行排序。 * @author zfg * */public class RDD2DataFrameByReflection {public static void main(String[] args) {SparkConf conf = new SparkConf().setMaster("local").setAppName("RDD2DataFrameByReflection");JavaSparkContext sc = new JavaSparkContext(conf);SQLContext sqlcontext = new SQLContext(sc);JavaRDD<String> lines = sc.textFile("Peoples.txt");JavaRDD<Person> personsRdd = lines.map(new Function<String, Person>() {/** *  */private static final long serialVersionUID = 1L;@Overridepublic Person call(String line) throws Exception {String[] split = line.split(",");Person p = new Person();p.setId(Integer.valueOf(split[0].trim()));p.setNam(split[1]);p.setAge(Integer.valueOf(split[2].trim()));return p;}});//傳入進去Person.class的時候，sqlContext是通過反射的方式建立DataFrame//在底層通過反射的方式或得Person的所有field，結合RDD本身，就產生了DataFrameDataFrame df = sqlcontext.createDataFrame(personsRdd, Person.class); //命名table的名字為persondf.registerTempTable("personTable");DataFrame resultDataFrame = sqlcontext.sql("select * from personTable where age > 7");resultDataFrame.show();//將df轉成rddJavaRDD<Row> resultRDD = resultDataFrame.javaRDD();JavaRDD<Person> result = resultRDD.map(new Function<Row, Person>() {@Overridepublic Person call(Row row) throws Exception { Person p = new Person(); p.setAge(row.getInt(0)); p.setId(row.getInt(1)); p.setNam(row.getString(2));return p;}}); List<Person> personList = result.collect();  for (Person person : personList) {System.out.println(person.toString());} }}