多任务分块下载器，支持断点续传

一个文件单独下载时，服务器有可能限速，造成下载时间过长。可通过多任务下载，提高下载速度。

分块下载

暂停+断点续传

// 测试下载 可能耗时多秒
var url = "http://cachefly.cachefly.net/10mb.test";
// 下载文件的名称
var name = WSCommFunc.GetRemoteName(url);
// 存储路径
var savePath = $"D:\\{WSCommFunc.RemoveInvalidFileNameChars(name)}";
var dir = new FileInfo(savePath).DirectoryName;
// 确保目录存在
System.IO.Directory.CreateDirectory(dir);

// 实例
var downloader = new MultiTaskDownloader(url, 10, savePath);
// 临时目录
downloader.TempFileDir = "D:\\";

CancellationTokenSource cts = new CancellationTokenSource(30000);
downloader.StartAsync(cts.Token).ConfigureAwait(false);
Thread.Sleep(2000);
downloader.Pause();
Thread.Sleep(3000);
downloader.ResumeAsync().ConfigureAwait(false);
Thread.Sleep(1000);
downloader.Cancel();

PrintThreadId("Main Thread End......");

下载2s后暂停，3s后继续运行

继续下载1s后取消

下载超时

......
CancellationTokenSource cts = new CancellationTokenSource(3000);
try
{
    downloader.StartAsync(cts.Token).ConfigureAwait(false).GetAwaiter().GetResult();
}
catch (Exception ex)
{
    Console.WriteLine("StartAsync: " + ex.Message);
    downloader.Cancel();
}

下载3s后超时退出

完整下载

......
downloader.StartAsync();

下载完成后合并

分块下载（待完善）

......
downloader.ChunkDownload();

多任务分块下载器 MultiTaskDownloader

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;

/// <summary>
/// 多任务分块下载器
/// </summary>
class MultiTaskDownloader
{
    /// <summary>
    /// 多线程数量
    /// </summary>
    public int NumberOfParts { get; private set; }
    /// <summary>
    /// 文件下载链接地址
    /// </summary>
    public string Url { get; set; }
    /// <summary>
    /// 下载文件存储路径
    /// </summary>
    private string TargetFilePath { get; set; }
    /// <summary>
    /// 临时文件夹
    /// </summary>
    public string TempFileDir { get; set; } = Environment.GetEnvironmentVariable("temp");

    /// <summary>
    /// 下载器集合
    /// </summary>
    private readonly List<PartialDownloader> _partialDownloaderList;
    /// <summary>
    /// 临时文件名称
    /// </summary>
    private string _tempFileName;
    /// <summary>
    /// 下载文件大小
    /// </summary>
    private long _contentLength;
    /// <summary>
    /// 协同取消操作
    /// </summary>
    private CancellationTokenSource _cts;
    public MultiTaskDownloader(string url, int numberOfParts, string savePath)
    {
        Url = url;

        if (WSCommFunc.IsRangeAllowed(url))
        {
            NumberOfParts = numberOfParts;
        }
        else
        {
            NumberOfParts = 1;
        }
        TargetFilePath = savePath;
        _partialDownloaderList = new List<PartialDownloader>();
    }

    /// <summary>
    /// 分块下载
    /// </summary>
    public async void ChunkDownload()
    {
        var req = (HttpWebRequest)WebRequest.Create(Url);
        req.UserAgent = WSCommFunc.UserAgent;
        req.AllowAutoRedirect = true;
        req.MaximumAutomaticRedirections = 5;
        req.ServicePoint.ConnectionLimit = 4;
        req.ServicePoint.Expect100Continue = true;
        // 1.1默认长连接，支持部分下载、分块传输
        req.ProtocolVersion = HttpVersion.Version11;
        req.SendChunked = true;
        req.TransferEncoding = "gzip";
        var timer = new HiPerfTimer();

        using (var response = (HttpWebResponse)req.GetResponse())
        {

        }

        using (var stream = await req.GetRequestStreamAsync())
        {

        }
    }

    /// <summary>
    /// 开始下载
    /// </summary>
    public async Task StartAsync()
    {
        _cts = new CancellationTokenSource();
        await StartAsync(_cts.Token);
    }

    /// <summary>
    /// 开始下载
    /// </summary>
    public async Task StartAsync(CancellationToken token)
    {
        if (_cts == null || _cts.Token != token)
        {
            WSCommFunc.Disponse(_cts);
            _cts = CancellationTokenSource.CreateLinkedTokenSource(token);
        }
        _contentLength = WSCommFunc.GetContentLength(Url);
        _tempFileName = GuidHelper.Increment().ToString() + "_tmp";
        var progress = new Progress<DownloaderProgressArg>(DownloaderProgressHandler);
        _partialDownloaderList.Clear();
        _taskDownloadBytes.Clear();
        for (int i = 0; i < NumberOfParts; i++)
        {
            var temp = CreatePartialDownloader(i, progress);
            _partialDownloaderList.Add(temp);
        }
        var listTasks = _partialDownloaderList.Select(s => s.DownladAsync(_cts.Token)).ToList();
        var timer = new HiPerfTimer();
        timer.Start();
        foreach (var item in WSCommFunc.Interleaved(listTasks))
        {
            var result = await item.ConfigureAwait(false);
            if (item.IsCompleted)
            {
                WSCommFunc.PrintThreadId($"{result.Index} Completed!");
            }
            else
            {
                WSCommFunc.PrintThreadId(item.Status.ToString());
            }
        }
        timer.Stop();
        var speed = _contentLength / 1024 / timer.Duration;
        Console.WriteLine($"{Url}: {timer.Duration} ms, Speed:{speed,5:f2} MB/S");
    }
    /// <summary>
    /// 暂停下载
    /// </summary>
    public void Pause()
    {
        WSCommFunc.PrintThreadId("Pause......");
        foreach (var item in _partialDownloaderList)
        {
            item.Stopped = true;
        }
    }
    public async Task ResumeAsync()
    {
        WSCommFunc.PrintThreadId("Resume......");
        var progress = new Progress<DownloaderProgressArg>(DownloaderProgressHandler);
        for (int i = 0; i < NumberOfParts; i++)
        {
            var temp = CreatePartialDownloader(i, progress);
            if (!temp.Completed)
            {
                _partialDownloaderList[i] = temp;
            }
        }
        WSCommFunc.Disponse(_cts);
        _cts = new CancellationTokenSource();
        var listTasks = _partialDownloaderList.Select(s => s.DownladAsync(_cts.Token)).ToList();
        var timer = new HiPerfTimer();
        timer.Start();
        foreach (var item in WSCommFunc.Interleaved(listTasks))
        {
            var result = await item.ConfigureAwait(false);
            if (item.IsCompleted)
            {
                WSCommFunc.PrintThreadId($"{result.Index} Completed!");
            }
            else
            {
                WSCommFunc.PrintThreadId(item.Status.ToString());
            }
        }
        timer.Stop();
        var speed = _contentLength / 1024 / timer.Duration;
        Console.WriteLine($"{Url}: {timer.Duration} ms, Speed:{speed,5:f2} MB/S");
    }

    /// <summary>
    /// 取消下载
    /// </summary>
    public void Cancel()
    {
        WSCommFunc.PrintThreadId("Cancel......");
        // 取消任务
        WSCommFunc.Disponse(_cts);
        // 减少中间过程产生的冗余文件
        _partialDownloaderList.AsParallel().ForAll(item =>
        {
            if (File.Exists(item.FullPath))
            {
                File.Delete(item.FullPath);
            }
        });
    }

    /// <summary>
    /// 多任务下载字节数
    /// </summary>
    private readonly ConcurrentDictionary<int, int> _taskDownloadBytes = new ConcurrentDictionary<int, int>();
    private static readonly object _objDownload = new object();
    private void DownloaderProgressHandler(DownloaderProgressArg arg)
    {
        // 多线程
        _taskDownloadBytes.AddOrUpdate(arg.Index, arg.BytesLength, (k, v) => v + arg.BytesLength);
        double percent = 0d;
        lock (_objDownload)
        {
            percent = _taskDownloadBytes.Sum(s => s.Value) * 1d / _contentLength;
            WSCommFunc.PrintThreadId($"{percent,6:P} {arg.Speed,10:f2}KB/s");
        }
        if (percent == 1.0)
        {
            Task.Run(async () =>
            {
                WSCommFunc.PrintThreadId("Merge......");
                var timer = new HiPerfTimer();
                timer.Start();
                await MergePartsAsync().ConfigureAwait(false);

                timer.Stop();
                WSCommFunc.PrintThreadId($"Merge...Completed! Size:{_contentLength} B, Time:{timer.Duration} ms ");
            });
        }
    }
    /// <summary>
    /// 按顺序合并分块文件
    /// </summary>
    /// <returns></returns>
    private async Task MergePartsAsync()
    {
        try
        {
            var orderList = _partialDownloaderList.OrderBy(x => x.PartialOrder);
            var dir = new FileInfo(TargetFilePath).DirectoryName;
            Directory.CreateDirectory(dir);
            using (var fs = new FileStream(TargetFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite))
            {
                //long totalBytes = 0;
                foreach (var item in orderList)
                {
                    using (var reader = File.OpenRead(item.FullPath))
                    {
                        byte[] buffer = new byte[81920];
                        int readLen;
                        while ((readLen = await reader.ReadAsync(buffer, 0, buffer.Length)) > 0)
                        {
                            await fs.WriteAsync(buffer, 0, readLen);
                            //totalBytes += readLen;
                            //var percent = totalBytes * 1d / ContentLength;
                            //Console.WriteLine($"Merge...{percent,5:P}");
                        }
                    }
                    //Thread.Sleep(0);
                    //File.Delete(item.FullPath);
                }
            }
            _partialDownloaderList.ForEach(itm =>
            {
                File.Delete(itm.FullPath);
            });
        }
        catch (Exception ex)
        {
            Console.WriteLine(ex.Message);
            throw;
        }

    }

    private PartialDownloader CreatePartialDownloader(int order, IProgress<DownloaderProgressArg> progress)
    {
        int division = (int)_contentLength / NumberOfParts;
        int remain = (int)_contentLength % NumberOfParts;
        int start = division * order;
        int end = start + division - 1;
        // 最后一块内容长度较大
        end += (order == NumberOfParts - 1 ? remain : 0);
        var fileName = Path.Combine(TempFileDir, _tempFileName) + order.ToString();
        var offset = 0;
        // 断点续传
        if (File.Exists(fileName))
        {
            offset = (int)new FileInfo(fileName).Length;
        }
        WSCommFunc.PrintThreadId($"{order} {fileName}");
        return new PartialDownloader(order, Url, fileName, start, end, offset, progress);
    }
}

部分下载 PartialDownloader

/// <summary>
/// 部分下载器
/// </summary>
class PartialDownloader
{
    /// <summary>
    /// 下载已完成
    /// </summary>
    public bool Completed { get; private set; }
    /// <summary>
    /// 部分索引
    /// </summary>
    public int PartialOrder { get; set; }
    /// <summary>
    /// 文件路径
    /// </summary>
    public string FullPath { get; set; }
    /// <summary>
    /// 下载已停止
    /// </summary>
    public bool Stopped { get; set; }
    /// <summary>
    /// 文件链接
    /// </summary>
    private readonly string _url;
    /// <summary>
    /// 源文件的起始位置
    /// </summary>
    private readonly int _from;
    /// <summary>
    /// 源文件的结束位置
    /// </summary>
    private readonly int _to;
    /// <summary>
    /// 下载内容的字节长度
    /// </summary>
    private readonly long _contentLength;
    /// <summary>
    /// 已下载内容的字节长度（支持断点续传）
    /// </summary>
    private int _totalLength;
    private readonly IProgress<DownloaderProgressArg> _progress;
    /// <summary>
    /// 文件下载器
    /// </summary>
    /// <param name="order">文件部分序号</param>
    /// <param name="url">文件链接</param>
    /// <param name="fileName">文件名称</param>
    /// <param name="start">需要读取的起始位置</param>
    /// <param name="end">需要读取的结束位置</param>
    /// <param name="offset">字节偏移量</param>
    /// <param name="progress">进度条</param>
    public PartialDownloader(int order, string url, string fileName, int start, int end, int offset, IProgress<DownloaderProgressArg> progress)
    {
        PartialOrder = order;
        FullPath = fileName;
        _url = url;
        _contentLength = end - start + 1;
        _from = start + offset;
        _to = end;
        _totalLength = offset;
        _progress = progress;
        //if (start >= end || start < 0 || end < 0)
        if (_from >= _to || _from < 0 || _to < 0)
        {
            Completed = true;
            Stopped = true;
        }
    }
    public async Task<DownloaderProgressArg> DownladAsync()
    {
        return await DownladAsync(CancellationToken.None);
    }
    // 考虑校验 分块已下载内容的合法性，不合法或已过期，需要重新下载
    // 压缩文件的处理 GZipStream
    // From、To 支持long类型
    public async Task<DownloaderProgressArg> DownladAsync(CancellationToken token)
    {
        var rslt = new DownloaderProgressArg() { Index = PartialOrder };
        if (token.IsCancellationRequested || Stopped)
        {
            WSCommFunc.PrintThreadId($"{PartialOrder} completed......");
            return rslt;
        }
        WSCommFunc.PrintThreadId($"{PartialOrder} starting......");
        Completed = false;
        try
        {
            using (var fs = new FileStream(FullPath, FileMode.Append, FileAccess.Write))
            {
                var req = (HttpWebRequest)WebRequest.Create(_url);
                req.UserAgent = WSCommFunc.UserAgent;
                req.AllowAutoRedirect = true;
                req.MaximumAutomaticRedirections = 5;
                req.ServicePoint.ConnectionLimit = 4;
                req.ServicePoint.Expect100Continue = true;
                // 1.1默认长连接，支持部分下载、分块传输
                req.ProtocolVersion = HttpVersion.Version11;
                req.AddRange(_from, _to);
                using (token.Register(() => { fs.Close(); req.Abort(); }, false))
                {
                    var timer = new HiPerfTimer();
                    int totalRead = 0;       // 总读取字节数
                    double totalCost = 0.0d; // 总耗时
                    using (var response = (HttpWebResponse)await req.GetResponseAsync())
                    {
                        using (var stream = response.GetResponseStream())
                        {
                            int readLength;
                            byte[] buffer = new byte[81920];
                            timer.Start();
                            while ((readLength = await stream.ReadAsync(buffer, 0, buffer.Length)) > 0)
                            {
                                timer.Stop();
                                totalCost += timer.Duration;
                                if (_totalLength + readLength > _contentLength)
                                {
                                    readLength = (int)(_contentLength - _totalLength);
                                }
                                await fs.WriteAsync(buffer, 0, readLength);
                                _totalLength += readLength;
                                totalRead += readLength;
                                var arg = new DownloaderProgressArg()
                                {
                                    Index = PartialOrder,
                                    BytesLength = readLength,
                                    Speed = totalRead / totalCost,
                                };
                                if (_totalLength == _contentLength)
                                {
                                    // 提前关闭，防止文件被占用
                                    fs.Close();
                                }
                                _progress.Report(arg);
                                if (Stopped)
                                {
                                    Completed = false;
                                    await fs.FlushAsync();
                                    fs.Close();
                                    req.Abort();
                                }
                                if (_totalLength == _contentLength)
                                {
                                    Completed = true;
                                    break;
                                }
                                timer.Start();
                            }
                        }
                    }
                }
                req.Abort();
            }
        }
        catch (WebException ex)
        {
            Console.WriteLine(ex.Message);
            if (token.IsCancellationRequested)
            {
                File.Delete(FullPath);
            }
            throw ex;
        }
        catch (Exception ex)
        {
            throw ex;
        }

        return rslt;
    }


}

下载进度参数类型 DownloaderProgressArg

/// <summary>
/// 下载进度参数类型
/// </summary>
class DownloaderProgressArg
{
    /// <summary>
    /// 索引（多任务下载时有用）
    /// </summary>
    public int Index { get; set; }
    /// <summary>
    /// 下载字节数
    /// </summary>
    public int BytesLength { get; set; }
    /// <summary>
    /// 下载速度（KB/s）
    /// </summary>
    public double Speed { get; set; }
}

问题解析

(503) 服务器不可用

服务器不堪重负，可以减少并发访问量。