Java讀寫avro例子，java讀寫avro

Java讀寫avro例子，java讀寫avro

一、avro是一個資料序列化架構，可以高效得進行序列化和還原序列化，支援C, C++, C#, Java, PHP, Python, 和Ruby語言。現在使用Java來讀寫。

二、環境搭建

　　1、下載avro-1.7.7.jar and avro-tools-1.7.7.jar兩個jar包，放到指定檔案目錄。 http://www.trieuvan.com/apache/avro/avro-1.7.7/java/

　　　　我放到了D:\soft\avro 檔案夾，在改目錄下建立java檔案夾，用來存放產生的Java代碼

　  2、該目錄下建立user.avsc檔案，內容是：　　

{"namespace": "example.avro", "type": "record", "name": "User", "fields": [     {"name": "name", "type": "string"},     {"name": "favorite_number",  "type": ["int", "null"]},     {"name": "favorite_color", "type": ["string", "null"]}

 ]}   

　　3、開啟cmd，進入到該目錄,執行命令產生User類

java -jar avro-tools-1.7.7.jar compile schema user.avsc java .

　　

　　在該檔案夾下的Java檔案下的../example/avro/目錄下就會產生User.java檔案。

　　4.使用eclipse建立maven項目，在pom.xml加入avro的依賴。　　

<dependency>  <groupId>org.apache.avro</groupId>  <artifactId>avro</artifactId>  <version>1.7.7</version></dependency>

三、產生的User.java檔案的內容如下。

　　把產生的User.java類複製到工程中，注意這個User.java裡面產生的User類及其內部類的包名預設是user.avsc文件中的namespace的值，

　　在本例中也就是example.avro。需要全部替換為自己的包名。

　　最簡單的方法就是把User.java中的example.avro全部替換為自己的包名。

/** * Autogenerated by Avro *  * DO NOT EDIT DIRECTLY */package example.avro;  @SuppressWarnings("all")@org.apache.avro.specific.AvroGeneratedpublic class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {  public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"example.avro\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}");  public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }  @Deprecated public java.lang.CharSequence name;  @Deprecated public java.lang.Integer favorite_number;  @Deprecated public java.lang.CharSequence favorite_color;  /**   * Default constructor.  Note that this does not initialize fields   * to their default values from the schema.  If that is desired then   * one should use <code>newBuilder()</code>.    */  public User() {}  /**   * All-args constructor.   */  public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color) {    this.name = name;    this.favorite_number = favorite_number;    this.favorite_color = favorite_color;  }  public org.apache.avro.Schema getSchema() { return SCHEMA$; }  // Used by DatumWriter.  Applications should not call.   public java.lang.Object get(int field$) {    switch (field$) {    case 0: return name;    case 1: return favorite_number;    case 2: return favorite_color;    default: throw new org.apache.avro.AvroRuntimeException("Bad index");    }  }  // Used by DatumReader.  Applications should not call.   @SuppressWarnings(value="unchecked")  public void put(int field$, java.lang.Object value$) {    switch (field$) {    case 0: name = (java.lang.CharSequence)value$; break;    case 1: favorite_number = (java.lang.Integer)value$; break;    case 2: favorite_color = (java.lang.CharSequence)value$; break;    default: throw new org.apache.avro.AvroRuntimeException("Bad index");    }  }  /**   * Gets the value of the 'name' field.   */  public java.lang.CharSequence getName() {    return name;  }  /**   * Sets the value of the 'name' field.   * @param value the value to set.   */  public void setName(java.lang.CharSequence value) {    this.name = value;  }  /**   * Gets the value of the 'favorite_number' field.   */  public java.lang.Integer getFavoriteNumber() {    return favorite_number;  }  /**   * Sets the value of the 'favorite_number' field.   * @param value the value to set.   */  public void setFavoriteNumber(java.lang.Integer value) {    this.favorite_number = value;  }  /**   * Gets the value of the 'favorite_color' field.   */  public java.lang.CharSequence getFavoriteColor() {    return favorite_color;  }  /**   * Sets the value of the 'favorite_color' field.   * @param value the value to set.   */  public void setFavoriteColor(java.lang.CharSequence value) {    this.favorite_color = value;  }  /** Creates a new User RecordBuilder */  public static example.avro.User.Builder newBuilder() {    return new example.avro.User.Builder();  }    /** Creates a new User RecordBuilder by copying an existing Builder */  public static example.avro.User.Builder newBuilder(example.avro.User.Builder other) {    return new example.avro.User.Builder(other);  }    /** Creates a new User RecordBuilder by copying an existing User instance */  public static example.avro.User.Builder newBuilder(example.avro.User other) {    return new example.avro.User.Builder(other);  }    /**   * RecordBuilder for User instances.   */  public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User>    implements org.apache.avro.data.RecordBuilder<User> {    private java.lang.CharSequence name;    private java.lang.Integer favorite_number;    private java.lang.CharSequence favorite_color;    /** Creates a new Builder */    private Builder() {      super(example.avro.User.SCHEMA$);    }        /** Creates a Builder by copying an existing Builder */    private Builder(example.avro.User.Builder other) {      super(other);      if (isValidValue(fields()[0], other.name)) {        this.name = data().deepCopy(fields()[0].schema(), other.name);        fieldSetFlags()[0] = true;      }      if (isValidValue(fields()[1], other.favorite_number)) {        this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);        fieldSetFlags()[1] = true;      }      if (isValidValue(fields()[2], other.favorite_color)) {        this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);        fieldSetFlags()[2] = true;      }    }        /** Creates a Builder by copying an existing User instance */    private Builder(example.avro.User other) {            super(example.avro.User.SCHEMA$);      if (isValidValue(fields()[0], other.name)) {        this.name = data().deepCopy(fields()[0].schema(), other.name);        fieldSetFlags()[0] = true;      }      if (isValidValue(fields()[1], other.favorite_number)) {        this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);        fieldSetFlags()[1] = true;      }      if (isValidValue(fields()[2], other.favorite_color)) {        this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);        fieldSetFlags()[2] = true;      }    }    /** Gets the value of the 'name' field */    public java.lang.CharSequence getName() {      return name;    }        /** Sets the value of the 'name' field */    public example.avro.User.Builder setName(java.lang.CharSequence value) {      validate(fields()[0], value);      this.name = value;      fieldSetFlags()[0] = true;      return this;     }        /** Checks whether the 'name' field has been set */    public boolean hasName() {      return fieldSetFlags()[0];    }        /** Clears the value of the 'name' field */    public example.avro.User.Builder clearName() {      name = null;      fieldSetFlags()[0] = false;      return this;    }    /** Gets the value of the 'favorite_number' field */    public java.lang.Integer getFavoriteNumber() {      return favorite_number;    }        /** Sets the value of the 'favorite_number' field */    public example.avro.User.Builder setFavoriteNumber(java.lang.Integer value) {      validate(fields()[1], value);      this.favorite_number = value;      fieldSetFlags()[1] = true;      return this;     }        /** Checks whether the 'favorite_number' field has been set */    public boolean hasFavoriteNumber() {      return fieldSetFlags()[1];    }        /** Clears the value of the 'favorite_number' field */    public example.avro.User.Builder clearFavoriteNumber() {      favorite_number = null;      fieldSetFlags()[1] = false;      return this;    }    /** Gets the value of the 'favorite_color' field */    public java.lang.CharSequence getFavoriteColor() {      return favorite_color;    }        /** Sets the value of the 'favorite_color' field */    public example.avro.User.Builder setFavoriteColor(java.lang.CharSequence value) {      validate(fields()[2], value);      this.favorite_color = value;      fieldSetFlags()[2] = true;      return this;     }        /** Checks whether the 'favorite_color' field has been set */    public boolean hasFavoriteColor() {      return fieldSetFlags()[2];    }        /** Clears the value of the 'favorite_color' field */    public example.avro.User.Builder clearFavoriteColor() {      favorite_color = null;      fieldSetFlags()[2] = false;      return this;    }    @Override    public User build() {      try {        User record = new User();        record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);        record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]);        record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]);        return record;      } catch (Exception e) {        throw new org.apache.avro.AvroRuntimeException(e);      }    }  }}

四、用Java實現序列化，即寫avro檔案。

　　建立一個Java類　　

　　　　 public static void main(String[] args) throws IOException {
　　　　 // 聲明並初始化User對象
　　　　 // 方式一
　　　　 User user1 = new User();        user1.setName("zhangsan");        user1.setFavoriteNumber(21);        user1.setFavoriteColor(null);        
　　　　 // 方式二 使用建構函式        // Alternate constructor        User user2 = new User("Ben", 7, "red");　　　　　　
 　　　　// 方式三，使用Build方式        // Construct via builder        User user3 = User.newBuilder()                     .setName("Charlie")                     .setFavoriteColor("blue")                     .setFavoriteNumber(null)                     .build();        String path = "D:\\tmp\\user.avro"; // avro檔案存放目錄        DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);        DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);        dataFileWriter.create(user1.getSchema(),  new File(path));
　　　　　// 把產生的user對象寫入到avro檔案        dataFileWriter.append(user1);        dataFileWriter.append(user2);        dataFileWriter.append(user3);        dataFileWriter.close();
}

　　run一下代碼，查看指定檔案目錄是否產生了avro檔案。

五、Java讀取avro檔案，即實現avro還原序列化。

public static void main(String[] args) throws IOException {       DatumReader<User> reader = new SpecificDatumReader<User>(User.class);       DataFileReader<User> dataFileReader = new DataFileReader<User>(new File("D:\\tmp\\user.avro"), reader);       User user = null;       while (dataFileReader.hasNext()) {            user = dataFileReader.next();            System.out.println(user);       }      }

   運行結果：

{"name": "zhangsan", "favorite_number": 21, "favorite_color": null}{"name": "Ben", "favorite_number": 7, "favorite_color": "red"}{"name": "Charlie", "favorite_number": null, "favorite_color": "blue"}

至此，例子寫完。