MySchedule.java 12.5 KB
package org.yrhl.syncdata.task;

import cn.hutool.core.bean.BeanUtil;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.yrhl.syncdata.domain.*;
import org.yrhl.syncdata.mapper.local.CourseOneMapper;
import org.yrhl.syncdata.mapper.remote.CourseTwoMapper;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@EnableScheduling //开启定时
@Component
public class MySchedule {
    private static final Logger log = LoggerFactory.getLogger(MySchedule.class);
    @Resource
    private CourseOneMapper courseOneMapper;
    @Resource
    private CourseTwoMapper courseTwoMapper;
    /**
     * 每隔10秒执行一次  执行生产者 准备数据
     */
    @Scheduled(fixedDelay = 20000)
    public void test(){
        //先查询需要同步的表信息
        List<SyncTable> syncTableList = courseOneMapper.getSyncTableList();
        if(syncTableList.isEmpty()){
            log.error("没有可用的同步表,请查看sync_table表是否有可用的数据");
            return ;
        }
        for(SyncTable syncTable:syncTableList){
            int i = insertIntoSyncResult(syncTable.getTableName());
            log.error("插入SyncResult状态为{}",i==1?"成功":"失败");
        }
    }


    @Scheduled(fixedDelay = 40000)
    public void test1() {
        //从中间表查询需要同步的数据
        List<SyncTable> syncTableList = courseOneMapper.getSyncTableList();
        for (SyncTable syncTable : syncTableList) {
            int i = syncTargetSoucre(syncTable.getTableName());
            log.error("数据同步到目标数据库{}",i==0?"成功":"失败");
        }

    }


    /***
     * 获取可备份的业务数据并将数据插入到SyncResult中待备份
     * @param tablename
     * @return
     */
    private  int insertIntoSyncResult(String tablename ) {

        int page = 1;
        while (true) {
            log.info("生产者:当前第{}页",page);
            //将数据插入到sync_result 中
            List<SyncResult> syncResultListForWait = null;
            if("allinonepushresults".equals(tablename)){
                syncResultListForWait = courseOneMapper.getSyncResultListForWaitForAllinonepushresults(tablename,(page-1)*100,100);
            }else{
                syncResultListForWait = courseOneMapper.getSyncResultListForWait(tablename,(page-1)*100,100);
            }
            if(CollectionUtils.isEmpty(syncResultListForWait)) {
                break;
            }
            try {
                // 1.在中间表查询是否存在数据(根据业务主键):如果存在,忽略,如果不存在,插入
                List<SyncResult> res= new ArrayList<>();
                for(SyncResult syncResult:syncResultListForWait){
                    // 遍历逐条分析:
                    int exesit = courseOneMapper.isExesit(syncResult.getInfoId());
                    if(0==exesit){
                        res.add(syncResult);
                    }
                }
                if(CollectionUtils.isNotEmpty(res)){
                    log.info("一共:"+res.size()+ "条");
                    long start = System.currentTimeMillis();
                    courseOneMapper.insertSyncResult(res);
                    long end = System.currentTimeMillis();
                    log.info("数据总耗时:" + (end-start) + "ms");
                }

            }catch (Exception e){
                log.error("插入数据失败", e);
                return 0;
            }
            page++;
        }

        return 0;
    }


    private int syncTargetSoucre(String tableName){

        if ("allinonepushresults".equals(tableName)) {
            int page = 1;
            while (true) {
                log.info("消费者:表名{}当前第{}页","allinonepushresults",page);
                List<AllinonepushresultsEntity> syncResultListForWait = courseOneMapper.getSyncResultListForWaitForConsumer(tableName,(page-1)*100,100);
                List<Long> ids = null;
                if (CollectionUtils.isEmpty(syncResultListForWait)) {
                    break;
                }
                try {
                    List<AllinonepushresultsEntity> resData = new ArrayList<>();
                    for (AllinonepushresultsEntity allinonepushresultsEntity : syncResultListForWait) {
                        int count = courseTwoMapper.isExist("allinonepushresults", allinonepushresultsEntity.getId());
                        if (0 == count) {
                            resData.add(allinonepushresultsEntity);
                        }
                    }
                    if (CollectionUtils.isNotEmpty(resData)) {
                        courseTwoMapper.insertAllinonepushresultsEntityBatch(resData);
                    }
                    ids = syncResultListForWait.stream()
                            .map(AllinonepushresultsEntity::getId)
                            .collect(Collectors.toList());
                    for (Long id : ids) {
                        courseOneMapper.updateSyncResult(id);
                    }
                } catch (Exception e) {
                    ids = syncResultListForWait.stream()
                            .map(AllinonepushresultsEntity::getId)
                            .collect(Collectors.toList());
                    for (Long id : ids) {
                        courseOneMapper.updateSyncResultFail(id);
                    }
                }
                page++;
            }
            log.info("消费者:表名{}同步完毕,一共{}页数据",tableName,page);
        }
        else if ("block_new_source_data".equals(tableName)) {
            int page = 1;
            while (true) {
                log.info("消费者:表名{}当前第{}页","block_new_source_data",page);
                List<Long> ids = null;
                List<Map<String, Object>> list3 = courseOneMapper.getSyncResultListForWaitForOtherConsumer(tableName,(page-1)*100,100);
                if (CollectionUtils.isEmpty(list3)) {
                    break;
                }
                List<BlockNewSourceDataEntity> res1 = list3.stream()
                        .map(BlockNewSourceDataEntity::fromMap)
                        .collect(Collectors.toList());
                page++;
                try {
                    List<BlockNewSourceDataEntity> resData = new ArrayList<>();
                    for (BlockNewSourceDataEntity blockNewSourceDataEntity : res1) {
                        int count = courseTwoMapper.isExist("block_new_source_data", blockNewSourceDataEntity.getId());
                        if (0 == count) {
                            resData.add(blockNewSourceDataEntity);
                        }
                    }
                    if (CollectionUtils.isNotEmpty(resData)) {
                        long start = System.currentTimeMillis();
                        courseTwoMapper.insertBlockNewSourceDataEntityBatch(resData);
                        long end = System.currentTimeMillis();
                        log.info("消费者插入数据总耗时:" + (end-start) + "ms");
                    }
                    ids = res1.stream()
                            .map(BlockNewSourceDataEntity::getId)
                            .collect(Collectors.toList());
                    for (Long id : ids) {
                        courseOneMapper.updateSyncResult(id);
                    }
                } catch (Exception e) {
                    ids = res1.stream()
                            .map(BlockNewSourceDataEntity::getId)
                            .collect(Collectors.toList());
                    for (Long id : ids) {
                        courseOneMapper.updateSyncResultFail(id);
                    }
                }
            }
            log.info("消费者:表名{}同步完毕,一共{}页数据",tableName);
        }
        else if("block_source_data".equals(tableName)){
            {
                int page = 1;
                while (true) {
                    log.info("消费者:表名{}当前第{}页","block_source_data",page);
                    List<Long> ids = null;
                    List<Map<String, Object>> list3 = courseOneMapper.getSyncResultListForWaitForOtherConsumer(tableName,(page-1)*100,100);
                    if (CollectionUtils.isEmpty(list3)) {
                        break;
                    }
                    List<BlockSourceDataEntity> res1 = list3.stream()
                            .map(BlockSourceDataEntity::fromMap)
                            .collect(Collectors.toList());
                    page++;
                    try {
                        List<BlockSourceDataEntity> resData = new ArrayList<>();
                        for (BlockSourceDataEntity blockSourceDataEntity : res1) {
                            int count = courseTwoMapper.isExist("block_source_data", blockSourceDataEntity.getId());
                            if (0 == count) {
                                resData.add(blockSourceDataEntity);
                            }
                        }
                        if (CollectionUtils.isNotEmpty(resData)) {
                            courseTwoMapper.insertBlockSourceDataEntityBatch(resData);
                        }
                        ids = res1.stream()
                                .map(BlockSourceDataEntity::getId)
                                .collect(Collectors.toList());
                        for (Long id : ids) {
                            courseOneMapper.updateSyncResult(id);
                        }
                    } catch (Exception e) {
                        ids = res1.stream()
                                .map(BlockSourceDataEntity::getId)
                                .collect(Collectors.toList());
                        for (Long id : ids) {
                            courseOneMapper.updateSyncResultFail(id);
                        }
                    }
                }
                log.info("消费者:表名{}同步完毕,一共{}页数据",tableName,page);
            }

        }
        else if("business_data_results".equals(tableName)){
            {
                int page = 1;
                while (true) {
                    log.info("消费者:表名{}当前第{}页","business_data_results",page);
                    List<Long> ids = null;
                    List<Map<String, Object>> list3 = courseOneMapper.getSyncResultListForWaitForOtherConsumer(tableName,(page-1)*100,100);
                    if (CollectionUtils.isEmpty(list3)) {
                        break;
                    }
                    List<BusinessDataResultsEntity> res1 = list3.stream()
                            .map(BusinessDataResultsEntity::fromMap)
                            .collect(Collectors.toList());
                    page++;
                    try {
                        List<BusinessDataResultsEntity> resData = new ArrayList<>();
                        for (BusinessDataResultsEntity businessDataResultsEntity : res1) {
                            int count = courseTwoMapper.isExist("business_data_results", businessDataResultsEntity.getId());
                            if (0 == count) {
                                resData.add(businessDataResultsEntity);
                            }
                        }
                        if (CollectionUtils.isNotEmpty(resData)) {
                            courseTwoMapper.insertBusinessDataResultsBatch(resData);
                        }
                        ids = res1.stream()
                                .map(BusinessDataResultsEntity::getId)
                                .collect(Collectors.toList());
                        for (Long id : ids) {
                            courseOneMapper.updateSyncResult(id);
                        }
                    } catch (Exception e) {
                        ids = res1.stream()
                                .map(BusinessDataResultsEntity::getId)
                                .collect(Collectors.toList());
                        for (Long id : ids) {
                            courseOneMapper.updateSyncResultFail(id);
                        }
                    }
                }
                log.info("消费者:表名{}同步完毕,一共{}页数据",tableName,page);
            }
        }

           return 0;
    }
}