Batch import data to Elasticsearch Based on Spring Batch, batchelasticsearch
1. Introduction
When the system imports a large amount of data from the database to Elasticsearch, using Spring Batch can improve the import efficiency. Spring Batch uses ItemReader to read data by PAGE and ItemWriter to write data in batches. Because Spring Batch does not provide the ItemWriter and ItemReader of Elastisearch, this example defines an ElasticsearchItemWriter (ElasticsearchItemReader) for Batch import.
2. Example
2.1 pom. xml
This article uses spring data jest to connect to ES (spring data elasticsearch can also be used to connect to ES). The ES version is 5.5.3.
<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.hfcsbc.estl</groupId> <artifactId>es-etl</artifactId> <version>0.0.1-SNAPSHOT</version> <packaging>jar</packaging> <name>es-etl</name> <description>Demo project for Spring Boot</description> <parent> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-parent</artifactId> <version>2.0.0.M7</version> <relativePath/> <!-- lookup parent from repository --> </parent> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <java.version>1.8</java.version> </properties> <dependencies> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter</artifactId> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> </dependency> <dependency> <groupId>org.postgresql</groupId> <artifactId>postgresql</artifactId> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-batch</artifactId> </dependency> <dependency> <groupId>com.github.vanroy</groupId> <artifactId>spring-boot-starter-data-jest</artifactId> <version>3.0.0.RELEASE</version> </dependency> <dependency> <groupId>io.searchbox</groupId> <artifactId>jest</artifactId> <version>5.3.2</version> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> </plugin> </plugins> </build> <repositories> <repository> <id>spring-snapshots</id> <name>Spring Snapshots</name> <url>https://repo.spring.io/snapshot</url> <snapshots> <enabled>true</enabled> </snapshots> </repository> <repository> <id>spring-milestones</id> <name>Spring Milestones</name> <url>https://repo.spring.io/milestone</url> <snapshots> <enabled>false</enabled> </snapshots> </repository> </repositories> <pluginRepositories> <pluginRepository> <id>spring-snapshots</id> <name>Spring Snapshots</name> <url>https://repo.spring.io/snapshot</url> <snapshots> <enabled>true</enabled> </snapshots> </pluginRepository> <pluginRepository> <id>spring-milestones</id> <name>Spring Milestones</name> <url>https://repo.spring.io/milestone</url> <snapshots> <enabled>false</enabled> </snapshots> </pluginRepository> </pluginRepositories></project>
2.2 entity class and repository
package com.hfcsbc.esetl.domain;import lombok.Data;import org.springframework.data.elasticsearch.annotations.Document;import org.springframework.data.elasticsearch.annotations.Field;import org.springframework.data.elasticsearch.annotations.FieldType;import javax.persistence.Entity;import javax.persistence.Id;import javax.persistence.OneToOne;/** * Create by pengchao on 2018/2/23 */@Document(indexName = "person", type = "person", shards = 1, replicas = 0, refreshInterval = "-1")@Entity@Datapublic class Person { @Id private Long id; private String name; @OneToOne @Field(type = FieldType.Nested) private Address address;}
package com.hfcsbc.esetl.domain;import lombok.Data;import javax.persistence.Entity;import javax.persistence.Id;/** * Create by pengchao on 2018/2/23 */@Entity@Datapublic class Address { @Id private Long id; private String name;}
package com.hfcsbc.esetl.repository.jpa;import com.hfcsbc.esetl.domain.Person;import org.springframework.data.jpa.repository.JpaRepository;/** * Create by pengchao on 2018/2/23 */public interface PersonRepository extends JpaRepository<Person, Long> {}
package com.hfcsbc.esetl.repository.es;import com.hfcsbc.esetl.domain.Person;import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;/** * Create by pengchao on 2018/2/23 */public interface EsPersonRepository extends ElasticsearchRepository<Person, Long> {}
2.3 configure elasticsearchItemWriter
Package com. hfcsbc. esetl. itemWriter; import com. hfcsbc. esetl. repository. es. esPersonRepository; import com. hfcsbc. esetl. domain. person; import org. springframework. batch. core. exitStatus; import org. springframework. batch. core. itemWriteListener; import org. springframework. batch. core. stepExecution; import org. springframework. batch. core. stepExecutionListener; import org. springframework. batch. item. itemWriter; impo Rt java. util. list;/*** Create by pengchao on example */public class ElasticsearchItemWriter implements ItemWriter <Person>, ItemWriteListener <Person>, StepExecutionListener {private comment personRepository; public comment (comment personRepository) {this. personRepository = personRepository;} @ Override public void beforeWrite (List <? Extends Person> items) {}@ Override public void afterWrite (List <? Extends Person> items) {}@ Override public void onWriteError (Exception exception Exception, List <? Extends Person> items) {}@ Override public void beforeStep (StepExecution stepExecution) {}@ Override public ExitStatus afterStep (StepExecution stepExecution) {return null ;} @ Override public void write (List <? Extends Person> items) throws Exception {// The saveAll method of the implementation class AbstractElasticsearchRepository calls elasticsearchOperations. bulkIndex (queries), which is a batch index of personRepository. saveAll (items );}}
2.4 configure ElasticsearchItemReader (this example is not used and is for reference only)
package com.hfcsbc.esetl.itemReader;import org.springframework.batch.item.data.AbstractPaginatedDataItemReader;import org.springframework.beans.factory.InitializingBean;import org.springframework.data.elasticsearch.core.ElasticsearchOperations;import org.springframework.data.elasticsearch.core.query.SearchQuery;import java.util.Iterator;/** * Create by pengchao on 2018/2/24 */public class ElasticsearchItemReader<Person> extends AbstractPaginatedDataItemReader<Person> implements InitializingBean { private final ElasticsearchOperations elasticsearchOperations; private final SearchQuery query; private final Class<? extends Person> targetType; public ElasticsearchItemReader(ElasticsearchOperations elasticsearchOperations, SearchQuery query, Class<? extends Person> targetType) { this.elasticsearchOperations = elasticsearchOperations; this.query = query; this.targetType = targetType; } @Override protected Iterator<Person> doPageRead() { return (Iterator<Person>)elasticsearchOperations.queryForList(query, targetType).iterator(); } @Override public void afterPropertiesSet() throws Exception { }}
2.5 configure spring batch
Package com. hfcsbc. esetl. config; import com. hfcsbc. esetl. itemWriter. elasticsearchItemWriter; import com. hfcsbc. esetl. repository. es. esPersonRepository; import com. hfcsbc. esetl. domain. person; import org. springframework. batch. core. job; import org. springframework. batch. core. step; import org. springframework. batch. core. configuration. annotation. enableBatchProcessing; import org. springframework. batch. core. configuration. annotation. jobBuilderFactory; import org. springframework. batch. core. configuration. annotation. stepBuilderFactory; import org. springframework. batch. core. launch. support. runIdIncrementer; import org. springframework. batch. core. repository. jobRepository; import org. springframework. batch. core. repository. support. jobRepositoryFactoryBean; import org. springframework. batch. item. itemReader; import org. springframework. batch. item. itemWriter; import org. springframework. batch. item. database. jpaPagingItemReader; import org. springframework. batch. item. database. orm. jpaNativeQueryProvider; import org. springframework. beans. factory. annotation. autowired; import org. springframework. context. annotation. bean; import org. springframework. context. annotation. configuration; import org. springframework. transaction. platformTransactionManager; import javax. persistence. entityManagerFactory; import javax. SQL. dataSource;/*** Create by pengchao on 2018/2/23 */@ Configuration @ your class BatchConfig {@ Autowired private extends personRepository; @ Bean public ItemReader <Person> orderItemReader (EntityManagerFactory entityManagerFactory) {JpaPagingItemReader <Person> reader = new JpaPagingItemReader <Person> (); String sqlQuery = "select * from person "; try {JpaNativeQueryProvider <Person> queryProvider = new JpaNativeQueryProvider <Person> (); queryProvider. setSqlQuery (sqlQuery); queryProvider. setEntityClass (Person. class); queryProvider. afterPropertiesSet (); reader. setEntityManagerFactory (entityManagerFactory); reader. setPageSize (10000); reader. setQueryProvider (queryProvider); reader. afterPropertiesSet (); reader. setSaveState (true);} catch (Exception e) {e. printStackTrace ();} return reader;} @ Bean public ElasticsearchItemWriter itemWriter () {return new ElasticsearchItemWriter (personRepository);} @ Bean public Step step (StepBuilderFactory stepBuilderFactory, ItemReader itemReader, ItemWriter itemWriter) {return stepBuilderFactory. get ("step1 "). chunks (10000 ). reader (itemReader ). writer (itemWriter ). build () ;}@ Bean public Job job (JobBuilderFactory jobBuilderFactory, Step step) {return jobBuilderFactory. get ("importJob "). incrementer (new RunIdIncrementer ()). flow (step ). end (). build ();}/*** some required tables will be created during spring batch execution. Here, the table creation location is specified: dataSource * @ param manager * @ return */@ Bean public JobRepository jobRepository (DataSource dataSource, PlatformTransactionManager manager) {specify JobRepositoryFactoryBean = new jobRepositoryFactoryBean (); response. setDataSource (dataSource); jobRepositoryFactoryBean. setTransactionManager (manager); jobRepositoryFactoryBean. setDatabaseType ("postgres"); try {return jobRepositoryFactoryBean. getObject ();} catch (Exception e) {e. printStackTrace ();} return null ;}}
2.6 configure the connection address of the database and es
spring: redis: host: 192.168.1.222 data: jest: uri: http://192.168.1.222:9200 username: elastic password: changeme jpa: database: POSTGRESQL show-sql: true hibernate: ddl-auto: update datasource: platform: postgres url: jdbc:postgresql://192.168.1.222:5433/person username: hfcb password: hfcb driver-class-name: org.postgresql.Driver max-active: 2spring.batch.initialize-schema: always
2.7 configuration Portal
package com.hfcsbc.esetl;import org.springframework.boot.SpringApplication;import org.springframework.boot.autoconfigure.SpringBootApplication;import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchAutoConfiguration;import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration;import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;import org.springframework.data.jpa.repository.config.EnableJpaRepositories;@SpringBootApplication(exclude = {ElasticsearchAutoConfiguration.class, ElasticsearchDataAutoConfiguration.class})@EnableElasticsearchRepositories(basePackages = "com.hfcsbc.esetl.repository")@EnableJpaRepositories(basePackages = "com.hfcsbc.esetl.repository.jpa")public class EsEtlApplication { public static void main(String[] args) { SpringApplication.run(EsEtlApplication.class, args); }}
The above is all the content of this article. I hope it will be helpful for your learning and support for helping customers.