Net WebClient 异步批量下载文件
1.文件提前使用爬虫整理好，放到了demo文件夹下的1.txt文件中，文件格式如：
http://www.banktunnel.eu/tumblr.com/tabea-lara.2_files/tumblr_n4kxopXkJ41sq93cpo10_1280.jpg
http://www.pptok.com/wp-content/uploads/2012/08/xunguang-9.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_3163ee0990499b7.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_25d8b5e402d0391.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFNWIA9vCAAmNhS2C8eAAAPyvQApsb4ACY2d098.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOGIIc0VAAom_BUlna4AAPyvQCLDjYACicU104.png
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOqIWH2cAAXWIl1DvA4AAPyvQDbhzkABdY6485.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFPGIfiurAAKxaF37_vcAAPyvQGII2YAArGA075.jpg
http://www.hr.com.cn/attachments/2015/06/b16af76b691f47609f3397e8cdd9c1e2.jpg
http://www.hr.com.cn/attachments/2015/06/226fc8acee0d478c9588ded1d04616b1.jpg
http://www.hr.com.cn/attachments/2015/06/c36e39b67d4c410cb794f553a271377b.jpg
2.新建DownloadFile类
public class DownloadFile
{
public DownloadFile()
{
}
public DownloadFile(string fileName, string fileFolder,string saveFileName)
{
FileName = fileName;
FileFolder = fileFolder;
SaveFileName = saveFileName;
}
public string FileName;
public string FileFolder;
public string SaveFileName;
}
3.Main函数创建下载用文件夹，读取需下载文件名称到List<DownloadFile>中。调用下载方法执行下载。
private static void Main(string[] args)
{
List<string> folders = new List<string>()
{
"demo","其他文件夹","其他文件夹1","xxxx"
};
Parallel.ForEach(folders, folder => CreateFolder(folder));
List<DownloadFile> downloadFiles = new List<DownloadFile>();
Parallel.ForEach(folders, folder =>
{
downloadFiles.AddRange(ReadFileUrl(folder));
});
Logger.Log(Level.Info, "开始任务执行");
List<Task> tList = new List<Task>();
downloadFiles.ForEach(p =>
{
tList.Add(
TransferWebFileHelper.DownloadingDataFromServerAsync(p)
);
});
Task.WaitAll(tList.ToArray());
Logger.Log(Level.Info, "任务执行结束");
}
/// <summary>
/// 创建下载用文件夹
/// </summary>
/// <param name="folderName"></param>
private static void CreateFolder(string folderName)
{
string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
if (!Directory.Exists(saveFolder))
{
Directory.CreateDirectory(saveFolder);
}
}
/// <summary>
/// 载入待下载文件到集合
/// </summary>
/// <param name="folderName"></param>
/// <returns></returns>
static List<DownloadFile> ReadFileUrl(string folderName)
{
List<DownloadFile> list = new List<DownloadFile>();
DirectoryInfo dir;
List<string> contents = new List<string>();
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName);
if (Directory.Exists(path))
{
dir = new DirectoryInfo(path);
FileInfo[] files = dir.GetFiles();
contents = File.ReadAllLines(files[0].FullName, Encoding.Default).ToList();
string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
contents.ForEach(fileName =>
{
//TODO 完全限定文件名必须少于 260 个字符，并且目录名必须少于 248 个字符。
string tempFileName = fileName.Split('?')[0].Replace("\\", "").Replace(":", "").Replace("*", "").Replace("\"", "").Replace("<", "").Replace(">", "").Replace("|", "");
int index = tempFileName.LastIndexOf("/");
tempFileName = tempFileName.Substring(index + 1, tempFileName.Length - index - 1);
int tempFileNameLength = tempFileName.Length;
if ((260 - saveFolder.Length - 2) < tempFileNameLength)
{
int difference = tempFileNameLength - (260 - saveFolder.Length - 2);//差
tempFileName = tempFileName.Substring(difference, tempFileName.Length - difference);
}
tempFileName = GetRandomStr() + "_" + tempFileName;
string saveFileName = Path.Combine(saveFolder, tempFileName);
var model = new DownloadFile(fileName, folderName, saveFileName);
list.Add(model);
});
}
return list;
}
4.下载文件类
public static class TransferWebFileHelper
{
/// <summary>
/// 新建文件，如果它不存在的话
/// </summary>
/// <param name="fileName"></param>
private static void CreateFile(string fileName)
{
if (!File.Exists(fileName))
{
FileStream fileStream = File.Create(fileName);
fileStream.Close();
fileStream.Dispose();
}
}
/// <summary>
/// 下载用方法
/// </summary>
/// <param name="downloadFile"></param>
/// <returns></returns>
public static async Task DownloadingDataFromServerAsync(DownloadFile downloadFile)
{
Uri uri = new Uri(downloadFile.FileName);
string saveFileName = downloadFile.SaveFileName;
CreateFile(saveFileName);
using (WebClient client = new WebClient())
{
try
{
await client.DownloadFileTaskAsync(uri, saveFileName);
}
catch (WebException ex)
{
Console.WriteLine($"从 {uri} 下载 {saveFileName}失败");
Logger.Log(Level.Info, $"从 {uri} 下载 {saveFileName}失败");
Logger.LogException(ex);
}
catch (Exception ex)
{
Console.WriteLine($"从 {uri} 下载 {saveFileName}失败");
Logger.Log(Level.Info, $"从 {uri} 下载 {saveFileName}失败");
Logger.LogException(ex);
}
}
}
}
5.其他辅助方法
private static string[] randomArray = new[]
{
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V",
"W", "X", "Y", "Z"
};
private static string GetRandomStr()
{
Random ran = new Random();
int i = ran.Next(0, 26);
Thread.Sleep(1);//尽量让生成的字符每一个都不相同
return randomArray[i];
}
