Import java.util.List;
Import org.apache.spark.SparkConf;
Import Org.apache.spark.api.java.JavaRDD;
Import Org.apache.spark.api.java.JavaSparkContext;
Import org.apache.spark.api.java.function.Function;
Import Org.apache.spark.sql.DataFrame;
Import Org.apache.spark.sql.Row;
Import Org.apache.spark.sql.SQLContext;
/** * Convert Rdd to Dataframe * 1, custom class must be public * 2, custom class must be serializable * 3, RDD when converted to Dataframe, he sorts according to the field names in the custom class. * @author ZFG * * */public class Rdd2dataframebyreflection {public static void main (string[] args) {sparkconf conf
= new sparkconf (). Setmaster ("local"). Setappname ("Rdd2dataframebyreflection");
Javasparkcontext sc = new Javasparkcontext (conf);
SqlContext sqlcontext = new SqlContext (SC);
Javardd<string> lines = Sc.textfile ("Peoples.txt"); javardd<person> Personsrdd = Lines.map (New function<string, person> () {/** * */Private St
Atic final Long serialversionuid = 1L; @Override Public person Call (StrinG line) throws Exception {string[] split = Line.split (",");
Person p = new person ();
P.setid (Integer.valueof (Split[0].trim ()));
P.setnam (split[1]);
P.setage (Integer.valueof (Split[2].trim ()));
return p;
}
}); When passed into the Person.class, SqlContext is created by reflection in Dataframe///At the bottom by reflection or by all the field of person, combining RDD itself to generate Dataframe dataframe
DF = Sqlcontext.createdataframe (Personsrdd, Person.class);
The name table is named person df.registertemptable ("persontable");
Dataframe resultdataframe = Sqlcontext.sql ("select * from persontable where age > 7");
Resultdataframe.show ();
Turn DF into rdd javardd<row> Resultrdd = Resultdataframe.javardd (); javardd<person> result = Resultrdd.map (new Function<row, person> () {@Override Public person call (Row R
ow) throws Exception {person p = new person ();
P.setage (row.getint (0));
P.setid (Row.getint (1));
P.setnam (row.getstring (2));
return p;
}
}); List<person> personlist = Result.collect ();
for (person person:personlist) {System.out.println (person.tostring ());
}
}
}