Memcached Enyim.Caching 解决无法遍历获取Key的问题

STATS命令

出于性能考虑，memcached没有提供遍历功能，不过我们可以通过以下两个stats命令得到所有的缓存对象。

1、stats items

     显示各个slab中item的数目。

2、stats cachedump slab_id limit_num
     显示某个slab中的前limit_num个key列表，显示格式：ITEM key_name [ value_length b; expire_time|access_time s]

基于此 我们就可以通过 扩展类的方式来实现其业务逻辑

    public class MemcachedClient : Enyim.Caching.MemcachedClient
    {
        public MemcachedClient(string sectionName) : base(sectionName)
        {

        }

        public IDictionary<IPEndPoint, List<string>> GetKeys()
        {
            var data = new Dictionary<IPEndPoint, List<string>>();
            var result = StatsItems("items");
            var itemsData = new Dictionary<IPEndPoint, Dictionary<string, string>>();
            foreach (var item in result)
            {
                var ip = item.Key;
                var items = new Dictionary<string, string>();
                foreach (var info in item.Value)
                {
                    var key = info.Key;
                    var strArry = key.Split(new string[] { ":" }, StringSplitOptions.RemoveEmptyEntries);
                    if (strArry.Length == 3 && strArry[2] == "number")
                    {
                        key = strArry[1].Trim();
                        var value = info.Value;
                        items.Add(key, value);
                    }
                }
                if (items.Any())
                {
                    itemsData.Add(ip, items);
                }
            }
            if (itemsData.Any())
            {
                foreach (var item in itemsData)
                {
                    var ip = item.Key;
                    var list = new List<string>();
                    foreach (var info in item.Value)
                    {
                        var dump = StatsItems($"cachedump {info.Key} {info.Value}");
                        if (dump.TryGetValue(ip, out Dictionary<string, string> dics))
                        {
                            foreach (var dic in dics)
                            {
                                list.Add(dic.Key);
                            }
                        }
                    }
                    if (list.Any())
                    {
                        data.Add(ip, list);
                    }
                }
            }
            return data;
        }
        private Dictionary<IPEndPoint, Dictionary<string, string>> StatsItems(string type)
        {
            var results = new Dictionary<IPEndPoint, Dictionary<string, string>>();

            foreach (var node in this.Pool.GetWorkingNodes())
            {
                IStatsOperation cmd = new StatsItemsOperation(type);
                node.Execute(cmd);
                results[node.EndPoint] = cmd.Result;
            }
            return results;
        }
    } 
    public class StatsItemsOperation : Operation, IStatsOperation
    {
        private static Enyim.Caching.ILog log = Enyim.Caching.LogManager.GetLogger(typeof(StatsOperation));

        private string type;
        private Dictionary<string, string> result;

        public StatsItemsOperation(string type)
        {
            this.type = type;
        }

        protected override IList<ArraySegment<byte>> GetBuffer()
        {
            var command = String.IsNullOrEmpty(this.type)
                            ? "stats" + TextSocketHelper.CommandTerminator
                            : "stats " + this.type + TextSocketHelper.CommandTerminator;

            return TextSocketHelper.GetCommandBuffer(command);
        }

        protected override IOperationResult ReadResponse(PooledSocket socket)
        {
            var serverData = new Dictionary<string, string>();

            while (true)
            {
                string line = TextSocketHelper.ReadResponse(socket);

                // stat values are terminated by END
                if (String.Compare(line, "END", StringComparison.Ordinal) == 0)
                    break;

                //// expected response is STAT item_name item_value
                //if (line.Length < 6 || String.Compare(line, 0, "STAT ", 0, 5, StringComparison.Ordinal) != 0)
                //{
                //	if (log.IsWarnEnabled)
                //		log.Warn("Unknow response: " + line);

                //	continue;
                //}

                // get the key&value
                string[] parts = line.Remove(0, 5).Split(' ');
                //if (parts.Length != 2)
                //{
                //	if (log.IsWarnEnabled)
                //		log.Warn("Unknow response: " + line);

                //	continue;
                //}

                // store the stat item
                serverData[parts[0]] = parts[1];
            }

            this.result = serverData;

            return new TextOperationResult().Pass();
        }

        Dictionary<string, string> IStatsOperation.Result
        {
            get { return result; }
        }

        protected override bool ReadResponseAsync(PooledSocket socket, System.Action<bool> next)
        {
            throw new System.NotSupportedException();
        }
    }
    internal static class TextSocketHelper
    {
        private const string GenericErrorResponse = "ERROR";
        private const string ClientErrorResponse = "CLIENT_ERROR ";
        private const string ServerErrorResponse = "SERVER_ERROR ";
        private const int ErrorResponseLength = 13;

        public const string CommandTerminator = "\r\n";

        private static readonly Enyim.Caching.ILog log = Enyim.Caching.LogManager.GetLogger(typeof(TextSocketHelper));

        /// <summary>
        /// Reads the response of the server.
        /// </summary>
        /// <returns>The data sent by the memcached server.</returns>
        /// <exception cref="T:System.InvalidOperationException">The server did not sent a response or an empty line was returned.</exception>
        /// <exception cref="T:Enyim.Caching.Memcached.MemcachedException">The server did not specified any reason just returned the string ERROR. - or - The server returned a SERVER_ERROR, in this case the Message of the exception is the message returned by the server.</exception>
        /// <exception cref="T:Enyim.Caching.Memcached.MemcachedClientException">The server did not recognize the request sent by the client. The Message of the exception is the message returned by the server.</exception>
        public static string ReadResponse(PooledSocket socket)
        {
            string response = TextSocketHelper.ReadLine(socket);

            if (log.IsDebugEnabled)
                log.Debug("Received response: " + response);

            if (String.IsNullOrEmpty(response))
                throw new MemcachedClientException("Empty response received.");

            if (String.Compare(response, GenericErrorResponse, StringComparison.Ordinal) == 0)
                throw new NotSupportedException("Operation is not supported by the server or the request was malformed. If the latter please report the bug to the developers.");

            if (response.Length >= ErrorResponseLength)
            {
                if (String.Compare(response, 0, ClientErrorResponse, 0, ErrorResponseLength, StringComparison.Ordinal) == 0)
                {
                    throw new MemcachedClientException(response.Remove(0, ErrorResponseLength));
                }
                else if (String.Compare(response, 0, ServerErrorResponse, 0, ErrorResponseLength, StringComparison.Ordinal) == 0)
                {
                    throw new MemcachedException(response.Remove(0, ErrorResponseLength));
                }
            }

            return response;
        }


        /// <summary>
        /// Reads a line from the socket. A line is terninated by \r\n.
        /// </summary>
        /// <returns></returns>
        private static string ReadLine(PooledSocket socket)
        {
            MemoryStream ms = new MemoryStream(50);

            bool gotR = false;
            //byte[] buffer = new byte[1];

            int data;

            while (true)
            {
                data = socket.ReadByte();

                if (data == 13)
                {
                    gotR = true;
                    continue;
                }

                if (gotR)
                {
                    if (data == 10)
                        break;

                    ms.WriteByte(13);

                    gotR = false;
                }

                ms.WriteByte((byte)data);
            }

            string retval = Encoding.ASCII.GetString(ms.GetBuffer(), 0, (int)ms.Length);

            if (log.IsDebugEnabled)
                log.Debug("ReadLine: " + retval);

            return retval;
        }

        /// <summary>
        /// Gets the bytes representing the specified command. returned buffer can be used to streamline multiple writes into one Write on the Socket
        /// using the <see cref="M:Enyim.Caching.Memcached.PooledSocket.Write(IList&lt;ArraySegment&lt;byte&gt;&gt;)"/>
        /// </summary>
        /// <param name="value">The command to be converted.</param>
        /// <returns>The buffer containing the bytes representing the command. The command must be terminated by \r\n.</returns>
        /// <remarks>The Nagle algorithm is disabled on the socket to speed things up, so it's recommended to convert a command into a buffer
        /// and use the <see cref="M:Enyim.Caching.Memcached.PooledSocket.Write(IList&lt;ArraySegment&lt;byte&gt;&gt;)"/> to send the command and the additional buffers in one transaction.</remarks>
        public  static IList<ArraySegment<byte>> GetCommandBuffer(string value)
        {
            var data = new ArraySegment<byte>(Encoding.ASCII.GetBytes(value));

            return new ArraySegment<byte>[] { data };
        }
    }

使用方式;

2022年4月2日 最新的方案

     
   /// <summary>
        /// 获取全部key
        /// </summary>
        /// <returns></returns>
        public List<string> GetKeys()
        {
            var data = new HashSet<string>();
            var itemsData = StatsItems("items");
            if (itemsData.Any())
            {
                foreach (var item in itemsData)
                {
                    var ip = item.Key;
                    foreach (var info in item.Value)
                    {
                        var dump = StatsItems($"cachedump {info} 0", ip);
                        if (dump.TryGetValue(ip, out var dics))
                        {
                            foreach (var dic in dics)
                            {
                                data.Add(dic);
                            }
                        }
                    }
                }
            }
            return data.ToList();
        }
        private Dictionary<IPEndPoint, List<string>> StatsItems(string type, IPEndPoint iPEndPoint = null)
        {
            var results = new Dictionary<IPEndPoint, List<string>>();

            var servers = Pool.GetWorkingNodes();
            if (iPEndPoint != null)
            {
                servers = servers.Where(t => t.EndPoint == iPEndPoint);
            }

            foreach (var node in servers)
            {
                IStatsOperation cmd = Pool.OperationFactory.Stats(type);
                try
                {
                    node.Execute(cmd);
                    results[node.EndPoint] = cmd.Result?.Keys.ToList();
                }
                catch (Exception e)
                {
                    logger.Fatal($"服务发生了异常:{node.EndPoint},{e.Message}", e);
                    throw;
                }
            }
            return results;
        }

2022年4月6日 cachedump 会丢失一部分数据

目前采用 lru_crawler metadump all 才能获取全部数据 -1.4.33 ++版本，需要开启 memcached -o lru_crawle