C#内存缓存链表BytesListBuffer

C#自带MemoryStream，可以作为内存缓存使用，用来存储byte[]数据，但是MemoryStream的扩展机制是通过获取整块连续内存来缓存数据，当需要缓存较大数据时，虽然空闲内存可能足够，但是可能找不到足够大的整块连续内存而导致扩展失败产生out of memory的异常。另外，对于很多缓存场景，重新分配整块内容，并将原缓存内容拷贝到新缓存，也会产生性能问题，而且使用链表缓存方式，通过将多块内存结合作为一个大缓存使用，能提升效率，也能解决大块内存不足问题。

BytesListBuffer将缓存作为内存块的链表保存，可以指定块大小，当所需缓存超过块大小时，会申请更多的块来保存数据，而原数据块保持不变。

BytesListBuffer继承自Stream，操作方法跟MemoryStream基本一致，读数据：

public override int Read(byte[] buffer, int offset, int count)
        {
            count = (int)Math.Min(_length - Position, count);

            int totalRead = 0;
            int copySize = 0;
            while (count > 0)
            {
                copySize = (int)Math.Min(count, BlockSize - blockPos);
                Buffer.BlockCopy(block, (int)blockPos, buffer, offset, copySize);

                Position += copySize;
                count -= copySize;
                offset += copySize;
                totalRead += copySize;
            }
            return totalRead;
        }

写数据：

public override void Write(byte[] buffer, int offset, int count)
        {
            if (BeforeWrite?.Invoke(count) == false)
                return;

            long beginPos = Position;
            int copySize;
            try
            {
                while (count > 0)
                {
                    copySize = Math.Min(count, (int)(BlockSize - blockPos));
                    Buffer.BlockCopy(buffer, offset, block, (int)blockPos, copySize);

                    _length = Math.Max(_length, Position + copySize);
                    Position += copySize;

                    count -= copySize;
                    offset += copySize;
                }
            }
            catch
            {
                Position = beginPos;
                throw;
            }
        }

默认块大小：

public long BlockSize = 64 * 1024;

默认块大小可以在对象初始化时根据需要调整，64K在大多数情况下有很好的效率。

完整代码：BytesListBuffer.cs

using System;
using System.Collections.Generic;
using System.IO;

namespace util
{
    public class BytesListBuffer : Stream
    {
        public long BlockSize = 64 * 1024;
        public List<byte[]> Blocks = new List<byte[]>();

        public Func<int, bool> BeforeWrite;

        public override bool CanRead => true;
        public override bool CanSeek => true;
        public override bool CanWrite => true;

        long _length = 0;
        public override long Length => _length;

        long _position = 0;
        public override long Position
        {
            get => _position;
            set => _position = value;
        }

        protected byte[] block
        {
            get
            {
                while (Blocks.Count <= blockIdx)
                {
                    Blocks.Add(new byte[BlockSize]);
                }
                return Blocks[(int)blockIdx];
            }
        }
        protected long blockIdx => Position / BlockSize;
        protected long blockPos => Position % BlockSize;

        public override void Flush() { }

        public override int Read(byte[] buffer, int offset, int count)
        {
            count = (int)Math.Min(_length - Position, count);

            int totalRead = 0;
            int copySize = 0;
            while (count > 0)
            {
                copySize = (int)Math.Min(count, BlockSize - blockPos);
                Buffer.BlockCopy(block, (int)blockPos, buffer, offset, copySize);

                Position += copySize;
                count -= copySize;
                offset += copySize;
                totalRead += copySize;
            }
            return totalRead;
        }

        public override void Write(byte[] buffer, int offset, int count)
        {
            if (BeforeWrite?.Invoke(count) == false)
                return;

            long beginPos = Position;
            int copySize;
            try
            {
                while (count > 0)
                {
                    copySize = Math.Min(count, (int)(BlockSize - blockPos));
                    Buffer.BlockCopy(buffer, offset, block, (int)blockPos, copySize);

                    _length = Math.Max(_length, Position + copySize);
                    Position += copySize;

                    count -= copySize;
                    offset += copySize;
                }
            }
            catch
            {
                Position = beginPos;
                throw;
            }
        }

        public override long Seek(long offset, SeekOrigin origin)
        {
            switch (origin)
            {
                case SeekOrigin.Begin:
                    Position = offset;
                    break;
                case SeekOrigin.Current:
                    Position += offset;
                    break;
                case SeekOrigin.End:
                    Position = Length - offset;
                    break;
            }
            return Position;
        }

        public override void SetLength(long value)
        {
            _length = value;
        }

        public byte[] ToArray()
        {
            long pos = Position;
            Position = 0;
            byte[] dst = new byte[Length];
            Read(dst, 0, (int)Length);
            Position = pos;
            return dst;
        }

        public void ReadFrom(Stream rd, long count)
        {
            byte[] buff = new byte[4096];
            int read;
            do
            {
                read = rd.Read(buff, 0, (int)Math.Min(4096, count));
                count -= read;
                this.Write(buff, 0, read);
            }
            while (count > 0);
        }

        public void WriteTo(Stream wrt)
        {
            long pos = Position;
            Position = 0;
            this.CopyTo(wrt);
            Position = pos;
        }

        //void check(byte[] buffer, int offset, int count)
        //{
        //    if (count < 0)
        //        throw new ArgumentOutOfRangeException();
        //    if (buffer == null)
        //        throw new ArgumentNullException();
        //    if (offset < 0)
        //        throw new ArgumentOutOfRangeException();
        //}
    }
}

Github链接：

https://github.com/bsmith-zhao/vfs/blob/main/util/BytesListBuffer.cs