RabbitMQ集成项目实现攻略数据同步到Elasticsearch

RabbitMQ集成项目实现攻略数据同步到Elasticsearch(8.20)

一、创建travel-common-rabbit模块

1、添加依赖

 <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-amqp</artifactId>
        </dependency>
    </dependencies>

2、新建配置类RabbitConfig

@Configuration
public class RabbitConfig {

    // 队列名称
    public static final String QUEUE_NAME = "travel_queue";
    // 交换机名称
    public static final String EXCHANGE_NAME = "travel_exchange";
    // 路由键
    public static final String ROUTING_KEY = "travel_routingkey";

    @Bean
    public Queue queue() {
        return new Queue(QUEUE_NAME,true,false,false);
    }

    @Bean
    public DirectExchange exchange() {
        return new DirectExchange(EXCHANGE_NAME,true,false);
    }

    @Bean
    public Binding MyBinding() {
        return new Binding(QUEUE_NAME,Binding.DestinationType.QUEUE,EXCHANGE_NAME,ROUTING_KEY,null);
    }
}

二、travel-modules-strategy模块

1、添加依赖

<dependency>
	<groupId>cn.wolfcode.wolf2w</groupId>
    <artifactId>travel-common-rabbit</artifactId>
    <version>3.6.2</version>
 </dependency>

2、编写配置信息

spring:
	rabbitmq:
        addresses: 192.168.222.129:5672,192.168.222.130:5672
        username: root
        password: root
        virtual-host: /

3、新增攻略时发送消息

   /**
     * 新增攻略
     */
    @RequiresPermissions("business:strategy:add")
    @Log(title = "攻略", businessType = BusinessType.INSERT)
    @PostMapping
    public R<?> add(Strategy strategy)
    {
        return toAjax(strategyService.insert(strategy));
    }
------------------------------------------------------------------------------------------------------
    @Autowired
    private AmqpTemplate amqpTemplate;

    @Override
    public int insert(Strategy strategy) {
        StrategyTheme theme = strategyThemeMapper.selectById(strategy.getThemeId());
        strategy.setThemeName(theme.getName());

        StrategyCatalog catalog = strategyCatalogMapper.selectById(strategy.getCatalogId());
        strategy.setCatalogName(catalog.getName());
        strategy.setDestId(catalog.getDestId());
        strategy.setDestName(catalog.getDestName());
        strategy.setCreateTime(new Date());

        Long isabroad = remoteDestinationService.isabroad(strategy.getDestId(), "inner").getData();
        strategy.setIsabroad(isabroad);

        strategy.setViewnum(0L);
        strategy.setReplynum(0L);
        strategy.setFavornum(0L);
        strategy.setSharenum(0L);
        strategy.setThumbsupnum(0L);
        baseMapper.insert(strategy);

        Long id = strategy.getId();
        String content = strategy.getContent().getContent();
        StrategyContent strategyContent = new StrategyContent();
        strategyContent.setId(id);
        strategyContent.setContent(content);
        int ret = strategyContentMapper.insert(strategyContent);
        // todo 发消息
        // 攻略对象JSON化变成消息
        String message = JSON.toJSONString(strategy);
        amqpTemplate.convertAndSend(RabbitConfig.EXCHANGE_NAME,RabbitConfig.ROUTING_KEY,message);
        return ret;
    }

三、travel-modules-search模块

1、添加依赖

    <dependency>
            <groupId>cn.wolfcode.wolf2w</groupId>
            <artifactId>travel-common-rabbit</artifactId>
            <version>3.6.2</version>
        </dependency>

2、编写配置信息

spring:
	rabbitmq:
        addresses: 192.168.222.129:5672,192.168.222.130:5672
        username: root
        password: root
        virtual-host: /

3、新建TravelRabbitListener

@Component
public class TravelRabbitListener {

    @Autowired
    private ElasticsearchClient client;
    @Autowired
    private StrategyEsServiceImpl strategyEsServiceImpl;

    /**
     * RabbitMQ 消息监听方法
     * 监听 travel_queue 队列，当有新消息进入队列时自动触发
     *
     * @param message 从队列中接收到的 JSON 格式的消息字符串
     * @throws IOException 写入 ES 时可能抛出的 IO 异常
     */
    @RabbitListener(queues = {RabbitConfig.QUEUE_NAME})
    public void receive(String message) throws IOException {
        // 1. 将 JSON 字符串反序列化为 Strategy 实体对象
        Strategy strategy = JSON.parseObject(message, Strategy.class);

        // 2. 创建 ES 文档对象，并将 Strategy 的属性拷贝到 ES 文档对象中
        StrategyEs es = new StrategyEs();
        BeanUtils.copyProperties(strategy, es);

        // 3. 将文档写入 Elasticsearch
        //    - index：指定写入的索引名称（从 service 中获取）
        //    - id：以策略 ID 作为文档的唯一标识
        //    - document：写入的文档内容
        client.index(i -> i.index(strategyEsServiceImpl.INDEX_NAME)
                            .id(es.getId().toString())
                            .document(es));
    }
}

四、第三方保障

1、使用Redis服务器保证消息处理成功

​ 生产者发消息前先存到Redis里，消费者处理消息后从Redis里移除，这两个动作是对于同一条消息的一组操作，若消息存入后长时间没有被移除，那么大概率是丢了。那么此时我们可以判定该条消息队列处理失败，就需要补发消息。我们还需要设定一个定时任务，定期去检查Redis中有没有超时未移除的消息，如果有就从Redis中获取该消息，重新补发，这样就保证了消息的可靠性投递。

生产者发消息 → 写入Redis（待确认）→ 发送到MQ
                                    ↓
消费者消费成功 → 从Redis移除（确认）
消费者消费失败/宕机 → Redis中记录保留
                                    ↓
定时任务扫描Redis → 发现超时未移除的消息 → 补发到MQ
                                    ↓
消费者再次消费（需保证幂等）

2、代码实战

（1）生产者发送消息前存入Redis

STRATEGY_RABBIT_MESSAGE("strategy_rabbit_message", -1),
--------------------------------------------------------------------------------------------------
    @Override
    public int insert(Strategy strategy) {
        StrategyTheme theme = strategyThemeMapper.selectById(strategy.getThemeId());
        strategy.setThemeName(theme.getName());

        StrategyCatalog catalog = strategyCatalogMapper.selectById(strategy.getCatalogId());
        strategy.setCatalogName(catalog.getName());
        strategy.setDestId(catalog.getDestId());
        strategy.setDestName(catalog.getDestName());
        strategy.setCreateTime(new Date());

        Long isabroad = remoteDestinationService.isabroad(strategy.getDestId(), "inner").getData();
        strategy.setIsabroad(isabroad);

        strategy.setViewnum(0L);
        strategy.setReplynum(0L);
        strategy.setFavornum(0L);
        strategy.setSharenum(0L);
        strategy.setThumbsupnum(0L);
        baseMapper.insert(strategy);

        Long id = strategy.getId();
        String content = strategy.getContent().getContent();
        StrategyContent strategyContent = new StrategyContent();
        strategyContent.setId(id);
        strategyContent.setContent(content);
        int ret = strategyContentMapper.insert(strategyContent);
        //todo 发消息
        String message = JSON.toJSONString(strategy);

        // ZSet类型，存储member（消息）和score（存入时间戳），member唯一不可重复，
        String key = RedisKeys.STRATEGY_RABBIT_MESSAGE.getPrefix();
        redisService.addCacheZset(key, message,System.currentTimeMillis());
        
        // 发送消息到MQ
        amqpTemplate.convertAndSend(RabbitConfig.EXCHANGE_NAME,RabbitConfig.ROUTING_KEY,message);
        // 插入文章表
        return ret;
    }

（2）消费者处理完消息后，从Redis中移除这条消息记录

@Component
public class TravelRabbitListener {
    @Autowired
    private ElasticsearchClient client;
    @Autowired
    private StrategyEsServiceImpl strategyEsServiceImpl;
    @Autowired
    private RedisService redisService;

    @RabbitListener(queues = {RabbitConfig.QUEUE_NAME})
    public void receive(String message) throws IOException {
        //  把 JSON 字符串反序列化为 Strategy 对象
        Strategy strategy = JSON.parseObject(message, Strategy.class);
        // 创建 ES 文档对象，把 Strategy 的属性拷贝过去
        StrategyEs es = new StrategyEs();
        BeanUtils.copyProperties(strategy, es);
        // client.index(i->i.index(INDEX_NAME).id(es.getId().toString()).document(es))
        //  将数据写入 ES 索引库（以攻略ID作为文档ID）
        client.index(i->i.index(strategyEsServiceImpl.INDEX_NAME)
                .id(es.getId().toString())
                .document(es));

        // 消费者处理完消息后，从 Redis 的 ZSet 中移除这条消息记录
        String key = RedisKeys.STRATEGY_RABBIT_MESSAGE.getPrefix();
        redisService.deleteCacheZsetValue(key,message);
    }

}

（3）设置检查任务，定期检查有没有消息超时没执行成功

①StrategyTask 定时任务

@Component("strategyTask")
public class StrategyTask {

    @Autowired
    private RemoteStrategyService remoteStrategyService;

    // 检查任务
    public void checkRabbitMessage(){
        remoteStrategyService.checkRabbitMessage(SecurityConstants.INNER);
    }
}

②RemoteStrategyService Feign接口

@FeignClient(contextId = "RemoteStrategyService", name = ServiceNameConstants.STRATEGY_SERVICE, fallbackFactory = RemoteStrategyFallbackFactory.class)
public interface                                                                           RemoteStrategyService {
    @PostMapping("/strategies/feign/checkRabbitMessage")
    R<?> checkRabbitMessage(@RequestHeader(SecurityConstants.FROM_SOURCE) String source);
}

③RemoteStrategyFallbackFactory 降级回调

@Component
public class RemoteStrategyFallbackFactory implements FallbackFactory<RemoteStrategyService> {
    private static final Logger log = LoggerFactory.getLogger(RemoteStrategyFallbackFactory.class);

    @Override
    public RemoteStrategyService create(Throwable throwable) {
        log.error("攻略服务调用失败:{}", throwable.getMessage());

        return new RemoteStrategyService() {      
            @Override
            public R<?> checkRabbitMessage(String source) {
                return R.fail(Lists.newArrayList(), "检查Rabbit消息失败:" + throwable.getMessage());
            }
        };
    }
}

④StrategyController

 @PostMapping("/feign/checkRabbitMessage")
    public R<?> checkRabbitMessage() {
        strategyService.checkRabbitMessage();
        return R.ok();

    }
----------------------------------------------------------------------------------------------------
// 检查在redis中存储超过1分钟没有处理的消息
@Override
public void checkRabbitMessage() {
    // 获取Redis中存储待确认消息的ZSet的key
    String key = RedisKeys.STRATEGY_RABBIT_MESSAGE.getPrefix();

    // 计算超时阈值：当前时间戳 - 1分钟（60秒 * 1000毫秒）
    // 即：score <= maxTime 的消息，说明存入时间已超过1分钟，视为超时未处理
    double maxTime = System.currentTimeMillis() - 1 * 60 * 1000;

    // 从ZSet中查询score在[0, maxTime]范围内的所有member（即超时未处理的消息）
    // rangeByScore会返回所有score <= maxTime的消息，也就是存入超过1分钟还没被消费者移除的消息
    Set<String> set = redisService.rangeByScore(key, 0, maxTime);

    // 如果存在超时未处理的消息，则逐条补发
    if (set != null && set.size() > 0) {
        for (String value : set) {
            // 将超时的消息重新发送到RabbitMQ，由消费者再次处理
            amqpTemplate.convertAndSend(RabbitConfig.EXCHANGE_NAME, RabbitConfig.ROUTING_KEY, value);
        }
    }
}

⑤添加定时任务