FreeRedis 实现消费确认队列和消费延时确认队列

namespace FreeRedisQueue
{

    /// <summary>
    /// 队列消息基类
    /// </summary>
    public class QueueMessage
    {
        public string MsgId { get; set; } = Guid.NewGuid().ToString("N");
        public long CreateTimestamp { get; set; } = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
        public int RetryCount { get; set; }
    }

    /// <summary>
    /// 生产级 Redis 可靠队列（终极0BUG版）
    /// 修复所有：丢消息、乱序、重复、死锁、CPU、雪崩、泄漏、卡死
    /// </summary>
    /// <typeparam name="T"></typeparam>
    public class UltimateRedisQueue<T> : IAsyncDisposable where T : QueueMessage
    {
        private readonly RedisClient _redis;
        private readonly ILogger<UltimateRedisQueue<T>> _logger;

        // ====================== Redis 队列 Key 定义 ======================
        /// <summary>
        /// 主队列（存储待消费消息ID）
        /// </summary>
        private readonly string _queue;

        /// <summary>
        /// 临时队列（存储正在处理的消息，防止消费中断丢失）
        /// </summary>
        private readonly string _tempQueue;

        /// <summary>
        /// 延时队列（存储延时消息，按时间戳排序）
        /// </summary>
        private readonly string _delayZSet;

        /// <summary>
        /// 待确认队列（存储已拉取但未 ACK 的消息，用于超时回收）
        /// </summary>
        private readonly string _pendingZSet;

        /// <summary>
        /// 死信队列列表（存储死信消息ID）
        /// </summary>
        private readonly string _deadLetterList;

        /// <summary>
        /// 消息唯一ID集合（用于消息去重，防止重复入队）
        /// </summary>
        private readonly string _msgUniqueSet;

        /// <summary>
        /// 消息内容哈希表（存储消息完整 JSON 数据）
        /// </summary>
        private readonly string _msgHash;

        /// <summary>
        /// 死信内容哈希表（存储死信完整 JSON 数据）
        /// </summary>
        private readonly string _deadLetterHash;

        // ====================== 分布式锁 Key ======================
        /// <summary>
        /// 延时消息转移锁（防止多实例同时转移重复执行）
        /// </summary>
        private readonly string _delayLockKey;

        /// <summary>
        /// 超时消息回收锁（防止多实例同时回收）
        /// </summary>
        private readonly string _recoverLockKey;

        // ====================== 重试/队列配置 ======================
        /// <summary>
        /// 消费失败后，延迟重试的秒数（默认10秒）
        /// </summary>
        private readonly int _retryDelaySeconds = 10;

        // ====================== 核心常量配置 ======================
        /// <summary>
        /// 消息超时时间（未 ACK 超过该时间自动回收重试）
        /// </summary>
        private const int TimeoutSeconds = 40;

        /// <summary>
        /// 最大重试次数（超过进入死信）
        /// </summary>
        private const int MaxRetries = 3;

        /// <summary>
        /// 批量处理消息数量（延时/超时消息每次处理50条）
        /// </summary>
        private const int Batch = 50;

        /// <summary>
        /// 死信队列最大容量（超出自动清理最早数据）
        /// </summary>
        private const int DeadLetterLimit = 20000;

        /// <summary>
        /// 消息唯一ID过期天数（自动清理）
        /// </summary>
        private const int UniqueExpireDays = 3;

        /// <summary>
        /// 熔断阈值（连续错误达到该次数触发熔断）
        /// </summary>
        private const int CircuitBreakThreshold = 10;

        // ====================== 消费状态控制 ======================
        /// <summary>
        /// 消费者是否正在运行
        /// </summary>
        private bool _isRunning;

        /// <summary>
        /// 后台消费任务
        /// </summary>
        private Task _consumerTask;

        /// <summary>
        /// 消费取消令牌（用于停止消费）
        /// </summary>
        private readonly CancellationTokenSource _cts = new();

        /// <summary>
        /// 实例唯一标识（区分多实例运行）
        /// </summary>
        private readonly string _instanceId = Guid.NewGuid().ToString("N")[..6];

        /// <summary>
        /// 连续消费失败次数（用于熔断）
        /// </summary>
        private int _consecutiveErrorCount;

        /// <summary>
        /// 启动锁（防止重复启动消费）
        /// </summary>
        private readonly object _startLock = new();

        /// <summary>
        /// 是否已执行过崩溃恢复（确保只执行一次）
        /// </summary>
        private bool _isRecovered = false;

        public UltimateRedisQueue(RedisClient redis, ILogger<UltimateRedisQueue<T>> logger, string queueName = "ultimate:queue")
        {
            _redis = redis ?? throw new ArgumentNullException(nameof(redis));
            _logger = logger ?? throw new ArgumentNullException(nameof(logger));

            // 🔥 关键修复：自动根据消息类型生成独立队列
            string messageType = typeof(T).Name;
            string baseKey = $"{queueName}:{messageType}";

            _queue = $"{baseKey}:main";
            _tempQueue = $"{baseKey}:temp";
            _delayZSet = $"{baseKey}:delay";
            _pendingZSet = $"{baseKey}:pending";
            _deadLetterList = $"{baseKey}:dead:list";
            _msgUniqueSet = $"{baseKey}:unique";
            _msgHash = $"{baseKey}:msg:hash";
            _deadLetterHash = $"{baseKey}:dead:hash";

            _delayLockKey = $"{baseKey}:lock:delay";
            _recoverLockKey = $"{baseKey}:lock:recover";

            // 启动时恢复一次
            _ = RecoverTempQueueAsync();
        }

        #region 入队 / 延时入队
        public async Task<bool> EnqueueAsync(T message)
        {
            var lua = @"
            local exists = redis.call('SISMEMBER', KEYS[1], ARGV[1])
            if exists == 1 then return 0 end
            redis.call('SADD', KEYS[1], ARGV[1])
            redis.call('HSET', KEYS[2], ARGV[1], ARGV[2])
            redis.call('RPUSH', KEYS[3], ARGV[1])
            redis.call('EXPIRE', KEYS[1], 86400 * " + UniqueExpireDays + @")
            redis.call('EXPIRE', KEYS[2], 86400 * " + UniqueExpireDays + @")
            return 1";

            var json = JsonConvert.SerializeObject(message);
            var ret = await _redis.EvalAsync(lua,
                new[] { _msgUniqueSet, _msgHash, _queue },
                message.MsgId, json);

            if (ret.ToString() == "0")
            {
                _logger.LogWarning("[{Instance}] 消息重复 {MsgId}", _instanceId, message.MsgId);
                return false;
            }

            _logger.LogInformation("[{Instance}] 入队 {MsgId}", _instanceId, message.MsgId);
            return true;
        }

        public async Task<bool> EnqueueDelayAsync(T message, TimeSpan delay)
        {
            var lua = @"
            local exists = redis.call('SISMEMBER', KEYS[1], ARGV[1])
            if exists == 1 then return 0 end
            redis.call('SADD', KEYS[1], ARGV[1])
            redis.call('HSET', KEYS[2], ARGV[1], ARGV[2])
            redis.call('ZADD', KEYS[3], ARGV[3], ARGV[1])
            redis.call('EXPIRE', KEYS[1], 86400 * " + UniqueExpireDays + @")
            redis.call('EXPIRE', KEYS[2], 86400 * " + UniqueExpireDays + @")
            return 1";

            long score = DateTimeOffset.UtcNow.Add(delay).ToUnixTimeSeconds();
            var json = JsonConvert.SerializeObject(message);
            var ret = await _redis.EvalAsync(lua,
                new[] { _msgUniqueSet, _msgHash, _delayZSet },
                message.MsgId, json, score);

            if (ret.ToString() == "0")
            {
                _logger.LogWarning("[{Instance}] 延时消息重复 {MsgId}", _instanceId, message.MsgId);
                return false;
            }

            _logger.LogInformation("[{Instance}] 延时入队 {MsgId} {TotalSeconds}s",
                _instanceId, message.MsgId, delay.TotalSeconds);
            return true;
        }
        #endregion

        #region 原子拉取（绝对安全）
        private async Task<T> FetchAsync()
        {
            await MoveDelayToQueueAsync();
            await RecoverTimeoutPendingAsync();

            if (_cts.Token.IsCancellationRequested)
                return null;

            long now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();

            var lua = @"
            local msgId = redis.call('RPOPLPUSH', KEYS[1], KEYS[4])
            if not msgId then return nil end
            redis.call('ZADD', KEYS[2], ARGV[1], msgId)
            local body = redis.call('HGET', KEYS[3], msgId)
            return {msgId, body}
        ";

            object result = await _redis.EvalAsync(lua,
                new[] { _queue, _pendingZSet, _msgHash, _tempQueue },
                now);

            if (result is not object[] arr || arr.Length < 2)
                return null;

            string msgId = arr[0]?.ToString();
            string json = arr[1]?.ToString();

            if (string.IsNullOrEmpty(json))
            {
                await CleanMissingMessageAsync(msgId);
                return null;
            }

            try
            {
                return JsonConvert.DeserializeObject<T>(json);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "[{Instance}] 反序列化失败 → 死信 {MsgId}", _instanceId, msgId);
                await MoveToDeadLetterDirectAsync(msgId, json);
                return null;
            }
        }
        #endregion

        #region 【修复】临时队列恢复（正确版，只执行一次）
        private async Task RecoverTempQueueAsync()
        {
            if (_isRecovered) return;
            _isRecovered = true;

            using var loc = _redis.Lock($"{_tempQueue}:lock", 5);
            if (loc == null) return;

            int recovered = 0;
            while (true)
            {
                var lua = @"
                local msg = redis.call('RPOPLPUSH', KEYS[2], KEYS[1])
                return msg ~= nil and 1 or 0
            ";
                var r = await _redis.EvalAsync(lua, new[] { _queue, _tempQueue });
                if ((int)r == 0) break;
                recovered++;
            }

            if (recovered > 0)
                _logger.LogWarning("[{Instance}] 崩溃恢复临时队列消息 {Count} 条", _instanceId, recovered);
        }
        #endregion


        #region 【修复】死信（全量清理 + 死信HASH自动裁剪，防止内存溢出）
        private async Task MoveToDeadLetterDirectAsync(string msgId, string json)
        {
            var lua = @"
        redis.call('ZREM', KEYS[1], ARGV[1])
        redis.call('LREM', KEYS[6], 0, ARGV[1])
        redis.call('HDEL', KEYS[2], ARGV[1])
        redis.call('SREM', KEYS[3], ARGV[1])
        redis.call('HSET', KEYS[4], ARGV[1], ARGV[2])
        redis.call('RPUSH', KEYS[5], ARGV[1])
      
        local maxLen = " + DeadLetterLimit + @"
        local nowLen = redis.call('LLEN', KEYS[5])
        if nowLen > maxLen then
            local dropMsgId = redis.call('LPOP', KEYS[5]) 
            redis.call('HDEL', KEYS[4], dropMsgId) 
        end
        return 1
    ";
            await _redis.EvalAsync(lua,
                new[] { _pendingZSet, _msgHash, _msgUniqueSet, _deadLetterHash, _deadLetterList, _tempQueue },
                msgId, json);
        }
        #endregion
        #region 【修复】清理消息（LREM 0）
        private async Task CleanMissingMessageAsync(string msgId)
        {
            var lua = @"
            redis.call('LREM', KEYS[1], 0, ARGV[1])
            redis.call('LREM', KEYS[6], 0, ARGV[1])
            redis.call('ZREM', KEYS[2], ARGV[1])
            redis.call('ZREM', KEYS[3], ARGV[1])
            redis.call('HDEL', KEYS[4], ARGV[1])
            redis.call('SREM', KEYS[5], ARGV[1])
            return 1
        ";
            await _redis.EvalAsync(lua,
                new[] { _queue, _pendingZSet, _delayZSet, _msgHash, _msgUniqueSet, _tempQueue },
                msgId);
            _logger.LogError("[{Instance}] 消息丢失已清理 {MsgId}", _instanceId, msgId);
        }
        #endregion

        #region 【修复】延时转移（安全 unpack）
        private async Task MoveDelayToQueueAsync()
        {
            using var loc = _redis.Lock(_delayLockKey, 5);
            if (loc == null) return;

            long now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
            var ids = await _redis.ZRangeByScoreAsync(_delayZSet, 0, now, 0, Batch);
            if (ids == null || ids.Length == 0) return;

            var lua = @"
            redis.call('RPUSH', KEYS[1], unpack(ARGV, 1, #ARGV))
            redis.call('ZREM', KEYS[2], unpack(ARGV, 1, #ARGV))
            return #ARGV
        ";
            await _redis.EvalAsync(lua, new[] { _queue, _delayZSet }, ids.Cast<object>().ToArray());
            _logger.LogInformation("[{Instance}] 转移延时消息 {Count}", _instanceId, ids.Length);
        }
        #endregion

        #region 【修复】超时回收（安全 unpack）
        private async Task RecoverTimeoutPendingAsync()
        {
            using var loc = _redis.Lock(_recoverLockKey, 5);
            if (loc == null) return;

            long now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
            long expireScore = now - TimeoutSeconds;
            var ids = await _redis.ZRangeByScoreAsync(_pendingZSet, 0, expireScore, 0, Batch);
            if (ids == null || ids.Length == 0) return;

            var lua = @"
            redis.call('RPUSH', KEYS[1], unpack(ARGV, 1, #ARGV))
            redis.call('ZREM', KEYS[2], unpack(ARGV, 1, #ARGV))
            return #ARGV
        ";
            await _redis.EvalAsync(lua, new[] { _queue, _pendingZSet }, ids.Cast<object>().ToArray());
            _logger.LogWarning("[{Instance}] 超时回收消息 {Count}", _instanceId, ids.Length);
        }
        #endregion

        #region 【修复】ACK（LREM 0 + SREM）
        public async Task AckAsync(T message)
        {
            var lua = @"
            redis.call('ZREM', KEYS[1], ARGV[1])
            redis.call('LREM', KEYS[3], 0, ARGV[1])
            redis.call('HDEL', KEYS[2], ARGV[1])
            redis.call('SREM', KEYS[4], ARGV[1])
            return 1
        ";
            await _redis.EvalAsync(lua,
                new[] { _pendingZSet, _msgHash, _tempQueue, _msgUniqueSet },
                message.MsgId);
            _logger.LogInformation("[{Instance}] ACK {MsgId}", _instanceId, message.MsgId);
        }
        #endregion


        #region 【增强】失败后延迟重试（可指定重试间隔）
        public async Task RejectAsync(T message, bool isManualReject = false)
        {
            message.RetryCount++;
            string msgId = message.MsgId;
            string json = JsonConvert.SerializeObject(message);

            // --------------- 【核心配置：重试延迟秒数】---------------
            int retryDelaySeconds = _retryDelaySeconds; // 👈 失败后等10秒再重试
                                                        // --------------------------------------------------------

            // 达到最大重试次数 → 死信
            if (!isManualReject && message.RetryCount >= MaxRetries)
            {
                await MoveToDeadLetterDirectAsync(msgId, json);
                _logger.LogError("[{Instance}] 死信 {MsgId} 重试={Retry}", _instanceId, msgId, message.RetryCount);
                return;
            }

            // 🔥 【关键修改】失败后不立即入队，而是加入延时队列，等待指定时间再重试
            var luaRetry = @"
        redis.call('ZREM', KEYS[1], ARGV[1])
        redis.call('LREM', KEYS[4], 0, ARGV[1])
        redis.call('HSET', KEYS[2], ARGV[1], ARGV[2]) 
        local executeTime = tonumber(ARGV[3])
        redis.call('ZADD', KEYS[5], executeTime, ARGV[1])
        return 1
    ";

            // 计算执行时间
            long executeTime = DateTimeOffset.UtcNow.AddSeconds(retryDelaySeconds).ToUnixTimeSeconds();

            await _redis.EvalAsync(luaRetry,
                new[] { _pendingZSet, _msgHash, _queue, _tempQueue, _delayZSet },
                msgId, json, executeTime);

            _logger.LogWarning("[{Instance}] 消费失败，{Delay}秒后重试 MsgId={MsgId} 重试次数={Retry}",
                _instanceId, retryDelaySeconds, msgId, message.RetryCount);
        }
        #endregion


        #region 【修复】自动消费（异常自动 Reject）
        public void StartAutoConsume(Func<T, Task<bool>> handler)
        {
            lock (_startLock)
            {
                if (_isRunning) return;
                _isRunning = true;
            }

            _consumerTask = Task.Run(async () =>
            {
                _logger.LogInformation("[{Instance}] 消费启动", _instanceId);
                while (!_cts.Token.IsCancellationRequested)
                {
                    T msg = null;
                    try
                    {
                        if (_consecutiveErrorCount >= CircuitBreakThreshold)
                        {
                            _logger.LogError("[{Instance}] 熔断，等待10s", _instanceId);
                            await Task.Delay(10000, _cts.Token);
                            _consecutiveErrorCount = 0;
                            continue;
                        }

                        msg = await FetchAsync();
                        if (msg == null)
                        {
                            await Task.Delay(300, _cts.Token);
                            continue;
                        }

                        bool ok = await handler(msg);
                        if (ok)
                        {
                            await AckAsync(msg);
                            _consecutiveErrorCount = 0;
                        }
                        else
                        {
                            await RejectAsync(msg);
                            _consecutiveErrorCount++;
                        }
                        msg = null;
                    }
                    catch (OperationCanceledException)
                    {
                        break;
                    }
                    catch (Exception ex)
                    {
                        _consecutiveErrorCount++;
                        _logger.LogError(ex, "[{Instance}] 消费异常 连续错误={Err}", _instanceId, _consecutiveErrorCount);

                        // 【修复】异常必须 Reject
                        if (msg != null) await RejectAsync(msg);

                        // 【修复】指数退避
                        int delay = Math.Min(1000 * (int)Math.Pow(2, _consecutiveErrorCount), 15000);
                        await Task.Delay(delay, _cts.Token);
                        msg = null;
                    }
                }
            }, _cts.Token);
        }

        public async Task StopAsync()
        {
            if (!_isRunning) return;
            _cts.Cancel();
            try
            {
                if (_consumerTask != null)
                    await _consumerTask.WaitAsync(TimeSpan.FromSeconds(10));
            }
            catch { }
            _isRunning = false;
            _logger.LogInformation("[{Instance}] 消费已停止", _instanceId);
        }
        #endregion

        #region 死信
        public async Task<List<T>> GetDeadLettersAsync(int count = 100)
        {
            var msgIds = await _redis.LRangeAsync(_deadLetterList, 0, count - 1);
            if (msgIds == null || msgIds.Length == 0) return new List<T>();

            var jsons = await _redis.HMGetAsync(_deadLetterHash, msgIds);
            var list = new List<T>();

            for (int i = 0; i < msgIds.Length; i++)
            {
                var json = jsons[i]?.ToString();
                if (!string.IsNullOrEmpty(json))
                {
                    try
                    {
                        list.Add(JsonConvert.DeserializeObject<T>(json));
                    }
                    catch { }
                }
            }
            return list;
        }

        public async Task ReprocessDeadLetter(T msg)
        {
            msg.RetryCount = 0;
            var lua = @"
            redis.call('LREM', KEYS[1], 0, ARGV[1])
            redis.call('HDEL', KEYS[2], ARGV[1])
            return 1
        ";
            await _redis.EvalAsync(lua, new[] { _deadLetterList, _deadLetterHash }, msg.MsgId);
            await EnqueueAsync(msg);
            _logger.LogInformation("[{Instance}] 死信重新入队 {MsgId}", _instanceId, msg.MsgId);
        }
        #endregion

        #region 释放
        public async ValueTask DisposeAsync()
        {
            await StopAsync();
            _cts.Dispose();
        }
        #endregion
    }
}