排查Hive報錯：org.apache.hadoop.hive.serde2.SerDeException: java.io.IOException: Start of Array expected

標籤：cal   create   exception   ges   pac   creat   oca   排查   路徑   

CREATE TABLE json_nested_test (    count string,    usage string,    pkg map<string,string>,    languages array<string>,    store map<string,array<map<string,string>>>)ROW FORMAT SERDE ‘org.apache.hive.hcatalog.data.JsonSerDe‘STORED AS TEXTFILE;

以上述sql建立表json_nested_test後，查詢時發現報錯：Failed with exception java.io.IOException:org.apache.hadoop.hive.serde2.SerDeException: java.io.IOException: Start of Array expected

於是進行如下測試，發現是org.apache.hive.hcatalog.data.JsonSerDe 對複雜類型支援不足造成，例如map<string,array<string>>，這個例子中就是不支援array作為map的value.

CREATE TABLE json_nested_test_openx (    count string,    usage string,    pkg map<string,string>,    languages array<string>,    store map<string,array<map<string,string>>>)ROW FORMAT SERDE ‘org.openx.data.jsonserde.JsonSerDe‘STORED AS TEXTFILE;CREATE TABLE s1 (    count string,    usage string)ROW FORMAT SERDE ‘org.apache.hive.hcatalog.data.JsonSerDe‘STORED AS TEXTFILE;load data local inpath ‘/home/work/s1.txt‘ overwrite into table s1;select * from s1;CREATE TABLE s2 (    count string,    usage string,    pkg map<string,string>)ROW FORMAT SERDE ‘org.apache.hive.hcatalog.data.JsonSerDe‘STORED AS TEXTFILE;load data local inpath ‘/home/work/s2.txt‘ overwrite into table s2;select * from s2;CREATE TABLE s3 (    count string,    usage string,    pkg map<string,string>,    languages array<string>)ROW FORMAT SERDE ‘org.apache.hive.hcatalog.data.JsonSerDe‘STORED AS TEXTFILE;load data local inpath ‘/home/work/s3.txt‘ overwrite into table s3;select * from s3;CREATE TABLE s4 (    count string,    usage string,    pkg map<string,string>,    languages array<string>,    store map<string,array<map<string,string>>>)ROW FORMAT SERDE ‘org.apache.hive.hcatalog.data.JsonSerDe‘STORED AS TEXTFILE;load data local inpath ‘/home/work/s4.txt‘ overwrite into table s4;select * from s4;CREATE TABLE s5 (    store map<string,array<map<string,string>>>)ROW FORMAT SERDE ‘org.apache.hive.hcatalog.data.JsonSerDe‘STORED AS TEXTFILE;load data local inpath ‘/home/work/s5.txt‘ overwrite into table s5;select * from s5;CREATE TABLE s6 (    store map<string,array<string>>)ROW FORMAT SERDE ‘org.apache.hive.hcatalog.data.JsonSerDe‘STORED AS TEXTFILE;load data local inpath ‘/home/work/s6.txt‘ overwrite into table s6;select * from s6;

這個Serde hive內建，路徑為$HIVE_HOME/hcatalog/share/hcatalog/hive-hcatalog-core-1.2.1.jar，它也存在其它問題：不支援資料檔案中的空行

排查Hive報錯：org.apache.hadoop.hive.serde2.SerDeException: java.io.IOException: Start of Array expected