Spring Boot 下使用 Spring Batch

公司有个小需求，就是将老平台用户信息和用户的资产信息迁移到新平台上。功能是实现起来是很简单。大概流程：

1.读取老平台用户

2.将读取到老平台用户信息转成新平台用户信息（还有其他的基本信息）的bean。

3.将bean写入到新平台。

 

这时突然想到了Spring Batch 框架，之前了解过，但一直没实践过。上面的需求感觉很适合用这个轻量级框架。

下面记录下，方便以后类似需求使用时参考。

说明：整个项目很小，使用的技术有： Spring Boot + Mybatis + Spring Batch 

因为要读取老平台数据库，插入到新平台数据库，所以这里使用了双数据源配置。

数据库密码是不可逆的，所以密码随机给、用户登录时根据状态提示用户必须修改密码才可以登录。

 

直接上干货：

 

pom.xml:

 

	<dependencies>
		<dependency>
			<groupId>org.mybatis.spring.boot</groupId>
			<artifactId>mybatis-spring-boot-starter</artifactId>
			<version>1.3.2</version>
		</dependency>

		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>
		
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-configuration-processor</artifactId>
			<optional>true</optional>
		</dependency>
		
		
		<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-batch</artifactId>
        </dependency>
	</dependencies>

 

application.properties:

 

datasource.news.driver-class-name=com.mysql.jdbc.Driver
datasource.news.jdbc-url=jdbc:mysql://127.0.0.1:3306/newtest?useUnicode=true&characterEncoding=UTF-8&useSSL=false
datasource.news.username=root
datasource.news.password=root




datasource.olds.driver-class-name=com.mysql.jdbc.Driver
datasource.olds.jdbc-url=jdbc:mysql://127.0.0.1:3306/oldtest?useUnicode=true&characterEncoding=UTF-8&useSSL=false
datasource.olds.username=root
datasource.olds.password=root




spring.batch.job.enabled=false




custom.houseId = 815300141843243
custom.levelId = 1

 

spring.batch.job.enabled = false 是启动项目不自动执行批任务

 

Spring Boot 启动类：

package com.coinex;

import org.mybatis.spring.boot.autoconfigure.MybatisAutoConfiguration;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;

@SpringBootApplication
@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class,DataSourceTransactionManagerAutoConfiguration.class, MybatisAutoConfiguration.class})
@EnableBatchProcessing
public class ExUdtsApplication {

	public static void main(String[] args) {
		SpringApplication.run(ExUdtsApplication.class, args);
	}
}

@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class,DataSourceTransactionManagerAutoConfiguration.class, MybatisAutoConfiguration.class})

加这个是因为数据库是双数据源，自动注入时会冲突。所以这里给去掉。

@EnableBatchProcessing     这个别忘记加了

 

 

二个数据源注解配置：

 

package com.coinex.config;

import javax.sql.DataSource;

import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.mybatis.spring.boot.autoconfigure.SpringBootVFS;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.PlatformTransactionManager;

@Configuration
@MapperScan(basePackages= {"com.coinex.news.mapper"},sqlSessionFactoryRef="newSqlSessionFactory")
public class NewDataSourceConfig {

	@Bean(name = "newDataSource")
	@ConfigurationProperties("datasource.news")
	public DataSource newDataSource() {
		return DataSourceBuilder.create().build();
	}
	
	@Bean(name = "newSqlSessionFactory")
	public SqlSessionFactory sqlSessionFactory(@Qualifier("newDataSource") DataSource dataSource) throws Exception {
		SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();
		sessionFactoryBean.setDataSource(dataSource);
		sessionFactoryBean.setVfs(SpringBootVFS.class);
		sessionFactoryBean.setTypeAliasesPackage("com.coinex.news.domain,com.coinex.news.entity");
		sessionFactoryBean.setMapperLocations(new PathMatchingResourcePatternResolver()
		        .getResources("classpath*:mapper/news/*.xml"));
		return sessionFactoryBean.getObject();
	}
	
	
	@Bean(name = "newTransactionManager")
	public PlatformTransactionManager prodTransactionManager(@Qualifier("newDataSource") DataSource dataSource) {
	    return new DataSourceTransactionManager(dataSource);
	}
}

package com.coinex.config;

import javax.sql.DataSource;

import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.mybatis.spring.boot.autoconfigure.SpringBootVFS;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;

@Configuration
@MapperScan(basePackages= {"com.coinex.olds.mapper"},sqlSessionFactoryRef="oldSqlSessionFactory")
public class OldDataSourceConfig {

	@Primary
	@Bean(name = "oldDataSource")
	@ConfigurationProperties("datasource.olds")
	public DataSource oldDataSource() {
		return DataSourceBuilder.create().build();
	}
	
	@Bean(name = "oldSqlSessionFactory")
	public SqlSessionFactory sqlSessionFactory(@Qualifier("oldDataSource") DataSource dataSource) throws Exception {
		SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();
		sessionFactoryBean.setDataSource(dataSource);
		sessionFactoryBean.setVfs(SpringBootVFS.class);
		sessionFactoryBean.setTypeAliasesPackage("com.coinex.olds.domain,com.coinex.olds.entity");
		sessionFactoryBean.setMapperLocations(new PathMatchingResourcePatternResolver()
		        .getResources("classpath*:mapper/olds/*.xml"));
		return sessionFactoryBean.getObject();
	}
}

sessionFactoryBean.setVfs(SpringBootVFS.class);  如果不加这个 把项目打成jar包运行时，会报mybatis找不到定义的别名类。

 

newTransactionManager  定义这个事物是为了让写入新库时用的。操作老平台库 和 新平台库不可能在一个事物内进行。

 

 

下面是Spring Batch 相关的代码：

首先简单说明下Spring Batch的几个组成部分：

一个Job有1个或多个Step组成，Step有读、处理、写三部分操作组成；通过JobLauncher启动Job，启动时从JobRepository获取Job Execution；当前运行的Job及Step的结果及状态保存在JobRepository中。

 

 

BatchConfig

package com.coinex.config;

import javax.batch.api.chunk.ItemReader;
import javax.sql.DataSource;

import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecutionListener;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.core.launch.support.SimpleJobLauncher;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.transaction.PlatformTransactionManager;

import com.coinex.core.listener.JobCompletionListener;
import com.coinex.core.setp.NewUserWriter;
import com.coinex.core.setp.OldUserReader;
import com.coinex.core.setp.UserProcessor;
import com.coinex.news.domain.UserMemberDo;
import com.coinex.olds.domain.FuserDo;

@Configuration
public class UserBatchConfig {

    @Autowired
    public JobBuilderFactory jobBuilderFactory;

    @Autowired
    public StepBuilderFactory stepBuilderFactory;
    
    
    
    
    /**
     * 创建JobRepository，用来注册Job的容器
     * @param dataSource
     * @param transactionManager
     * @return
     * @throws Exception
     */
/*    @Bean(name="myJobRepository")
    public JobRepository jobRepository(DataSource dataSource, PlatformTransactionManager transactionManager) throws Exception {
    	 JobRepositoryFactoryBean jobRepositoryFactoryBean =
                 new JobRepositoryFactoryBean();
         jobRepositoryFactoryBean.setDataSource(dataSource);
         jobRepositoryFactoryBean.setTransactionManager(transactionManager);
         jobRepositoryFactoryBean.setDatabaseType("mysql");
         return jobRepositoryFactoryBean.getObject();
    }*/
    
    
    /**
     * 创建 JobLauncher，用来启动Job的接口
     * @param dataSource
     * @param transactionManager
     * @return
     * @throws Exception
     */
/*    @Bean(name="myJobLauncher")
    public SimpleJobLauncher jobLauncher(DataSource dataSource,PlatformTransactionManager transactionManager)throws Exception{
        SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
        jobLauncher.setJobRepository(jobRepository(dataSource, transactionManager));
        return jobLauncher;
    }*/
    
    
    
    

    
    /**
     * 创建Job
     * @return
     */
    @Bean
    public Job processJob() {
    	return jobBuilderFactory.get("processJob")
    			.incrementer(new RunIdIncrementer())
    			.listener(listener())
    			.flow(userStep())
    			.end()
    			.build();
    }
    
    
    
    /**
     * 创建Step
     * @return
     */
    @Bean
    public Step userStep() {
    	return stepBuilderFactory.get("userStep").<FuserDo,UserMemberDo>chunk(100)
    			.reader(reader())
    			.processor(processor())
    			.writer(writer())
    			.build();
    			
    }
    
    /**
     * 创建Reader
     * @return
     */
    @Bean
    public OldUserReader reader() {
    	return  new OldUserReader();
    }
    
    /**
     * 创建Processor
     * @return
     */
    @Bean
    public UserProcessor processor() {
    	return new UserProcessor();
    }
    
    
    /**
     * 创建Writer
     * @return
     */
    @Bean
    public NewUserWriter writer() {
    	return new NewUserWriter();
    }
    
    /**
     * 创建Listener
     * @return
     */
    @Bean
    public JobExecutionListener listener() {
    	return new JobCompletionListener();
    }
    
}

 

 

Reader:

package com.coinex.core.setp;

import java.util.List;

import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.NonTransientResourceException;
import org.springframework.batch.item.ParseException;
import org.springframework.batch.item.UnexpectedInputException;
import org.springframework.beans.factory.annotation.Autowired;

import com.coinex.olds.domain.FuserDo;
import com.coinex.olds.entity.VritualWallet;
import com.coinex.olds.service.FuserService;

public class OldUserReader implements ItemReader<FuserDo> {
	
	@Autowired
	FuserService fuserService;
	
	@Override
	public FuserDo read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
		FuserDo fuserDo =  fuserService.getPendingUser();
		if(fuserDo != null) {
			//查询用户资产
			List<VritualWallet> wallets = fuserService.findUserVirtualWalletsByUid(fuserDo.getFid());
			fuserDo.setWallets(wallets);
			fuserService.setProcessedUser(fuserDo.getFid());
		}
		return fuserDo;
	}

}

 

 

Processor:

package com.coinex.core.setp;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.springframework.batch.item.ItemProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;

import com.coinex.common.UuidUtil;
import com.coinex.news.domain.UserMemberDo;
import com.coinex.news.entity.CoinInfor;
import com.coinex.news.entity.UserAssetInfo;
import com.coinex.news.entity.UserMemberExtend;
import com.coinex.news.service.UserMemberService;
import com.coinex.olds.domain.FuserDo;
import com.coinex.olds.entity.VritualWallet;

public class UserProcessor implements ItemProcessor<FuserDo, UserMemberDo> {
	
	
	@Value("${custom.houseId}")
	private Long houseId;
	
	
	@Value("${custom.levelId}")
	private Long levelId;
	
	@Autowired
	private UserMemberService userMemberService;

	@Override
	public UserMemberDo process(FuserDo oldUser) throws Exception {
		//将老用户有用的信息转转换到新用户实体上，密码随机
		UserMemberDo memberDo = new UserMemberDo();
		memberDo.setId(UuidUtil.getUuid());
		memberDo.setUsername(oldUser.getFloginName());
		memberDo.setEmail(oldUser.getFloginName());
		memberDo.setHouseId(houseId);
		memberDo.setApplyTime(new Date());
		memberDo.setEmailStatus(1);
		memberDo.setIdentifiStatus(0);
		memberDo.setLastUpdate(new Date());
		memberDo.setStatus(1);
		memberDo.setType(2);
		memberDo.setVersion(0);
		memberDo.setUserLevel(0);
		memberDo.setLevelId(levelId);
		memberDo.setFeeSwitch(0);
		
		//为新用户生成所有币种资产信息,并将老用户资产转换到新用户资产上
		List<CoinInfor> coinInfors = userMemberService.finCoinInfoByHouseId(houseId);
		
		List<UserAssetInfo> assetInfos = new ArrayList<UserAssetInfo>();
		
		List<VritualWallet> wallets = oldUser.getWallets();
		
		//这里有一个问题，就是当老平台有的币、新平台确没有这个币，则会导致用户这个币的资产落空！！！
		coinInfors.forEach( coinInfor -> {
			UserAssetInfo assetInfo = new UserAssetInfo();
			assetInfo.setId(UuidUtil.getUuid());
			assetInfo.setHouseId(houseId);
			assetInfo.setCoinType(coinInfor.getId());
			assetInfo.setFrozen(new BigDecimal(0));
			assetInfo.setLastTime(new Date());
			assetInfo.setMemId(memberDo.getId());
			assetInfo.setMemUsername(memberDo.getUsername());
			assetInfo.setTotal(getOldTotal(wallets,coinInfor.getSortName()));
			assetInfo.setVersion(0);
			assetInfo.setCoinName(coinInfor.getSortName());
			assetInfos.add(assetInfo);
		});
		
		memberDo.setAssetInfos(assetInfos);
		
		//用户扩展表
		UserMemberExtend memberExtend = new UserMemberExtend();
		memberExtend.setHouseId(houseId);
		memberExtend.setId(memberDo.getId());
		memberExtend.setCreateTime(new Date());
		memberExtend.setDealStatus(1);
		memberExtend.setRechargeStatus(1);
		memberExtend.setWithdrawStatus(1);
		memberExtend.setGooleStauts(0);
		memberExtend.setLoginOpenGA(0);
		memberExtend.setWithdrawOpenGA(0);
		memberExtend.setNoteAuthStatus(0);
		memberExtend.setStatus(memberDo.getStatus());
		memberExtend.setType(memberDo.getType());
		memberExtend.setWithdrawOpenNote(0);
		memberExtend.setTreePath(memberDo.getId()+"");
		memberExtend.setRegisterPath(2);
		memberExtend.setLastLogin(new Date());
		memberExtend.setInvalidPassword(1);
		memberDo.setMemberExtend(memberExtend);
		
		return memberDo;
	}
	
	
	
	/**
	 * 获取老平台用户资产total
	 * @param wallets
	 * @param sortName
	 * @return
	 */
	private BigDecimal getOldTotal(List<VritualWallet> wallets,String sortName) {
		for(VritualWallet wallet : wallets) {
			if(sortName.trim().equalsIgnoreCase(wallet.getCoinName().trim())) {
				return wallet.getTotal();
			}
		}
		return new BigDecimal(0);
	}
	

}

 

Writer:

package com.coinex.core.setp;

import java.util.List;
import java.util.concurrent.atomic.AtomicLong;

import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;

import com.coinex.news.domain.UserMemberDo;
import com.coinex.news.service.UserMemberService;

public class NewUserWriter implements ItemWriter<UserMemberDo> {

	@Autowired
	private UserMemberService userMemberService;
	
	private AtomicLong counter = new AtomicLong(0);  
	
	@Override
	public void write(List<? extends UserMemberDo> userMemberDos) throws Exception {
		userMemberDos.forEach( user -> {
			userMemberService.save(user);
			counter.incrementAndGet();
		});
		System.out.println("一个同步了："+counter);
	}

}

 

 

这里的save方法上要用newTransactionManager 事物 如：@Transactional(propagation=Propagation.REQUIRES_NEW,transactionManager="newTransactionManager")。要不save方法体会没事物的。注意，如果在save这块报错，Reader 和 Processos整个一批操作都会被回滚掉，因为整个操作在在old事物中进行的(数据源那默认设置oldDataSource)。当然，如果save方法执行完后框架内部报错了，那就会出现Reader 被回滚，但save的数据还是插入了，这是因为用了双数据源导致的。如果整个处理业务在一个数据源环境下操作Spring Batch会保证一批处理在一个事物内完成。（注：这里的一批是 前面batch config配置的  chunk(100)）

 

 

Listeners：

package com.coinex.core.listener;


import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.listener.JobExecutionListenerSupport;

public class JobCompletionListener extends JobExecutionListenerSupport {

	@Override
	public void afterJob(JobExecution jobExecution) {
		if(BatchStatus.COMPLETED == jobExecution.getStatus()) {
			System.out.println("old2new 任务执行完毕");
		}
	}

	@Override
	public void beforeJob(JobExecution jobExecution) {
		System.out.println("old2new 任务开始执行....");
	}

}

 

 

最后还要创建几张Spring Beach 内部用的几张表：

sql文件在 spring-batch-core-4.0.1 REEASE.jar 包里。根据自己的数据库去执行就可以了。

 

 

测试执行：

 

package com.coinex;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)
@SpringBootTest
public class BathTest {

	@Autowired
    JobLauncher jobLauncher;
 
    @Autowired
    Job processJob;
    
    @Test
    public void excutorTest() throws JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException {
        JobParameters jobParameters = new JobParametersBuilder()
        		.addLong("time", System.currentTimeMillis())
                .toJobParameters();
        jobLauncher.run(processJob, jobParameters);
    }
}