封装多线程处理大量数据操作（二）
接上文：封装多线程处理大量数据操作（一）
我们需要解决WaitAny和取得异步执行的返回值的问题。地球人都知道Thread和ThreadPool接受的委托都是没有返回值的。要想取的返回值，我们就得自己动手了，我们需要构造一个AsyncContext类，由这个类来保存异步执行的状态以并存储返回值。
代码如下：
using System;
using System.Collections.Generic;
using System.Text;
using System.Collections;
using System.Threading;
using System.Diagnostics;
namespace AppUtility
{
public delegate object DoGetObjTask(object state);
public static class AsyncHelper
{
/// <summary>
/// 执行多线程操作任务
/// </summary>
/// <param name="dataCollection">多线程操作的数据集合</param>
/// <param name="threadCn">分多少个线程来做</param>
/// <param name="processItemMethod">处理数据集合中单个数据使用的处理方法</param>
public static void DoAsync(IList dataCollection, int threadCn, WaitCallback processItemMethod)
{
DoAsync(dataCollection, threadCn, processItemMethod, true);
}
/// <summary>
/// 执行多线程操作任务
/// </summary>
/// <param name="dataCollection">多线程操作的数据集合</param>
/// <param name="threadCn">分多少个线程来做</param>
/// <param name="processItemMethod">处理数据集合中单个数据使用的处理方法</param>
/// <param name="needWaitAll">是否需要等待所有线程执行完毕才返回，为true时会等待所有线程执行完毕，否则则是在有一个线程执行完毕就返回</param>
public static void DoAsync(IList dataCollection, int threadCn, DoGetObjTask processItemMethod, bool needWaitAll, out Hashtable processResult)
{
DoAsyncPrivate(dataCollection, threadCn, null, processItemMethod, needWaitAll, true, out processResult);
}
/// <summary>
/// 执行多线程操作任务
/// </summary>
/// <param name="dataCollection">多线程操作的数据集合</param>
/// <param name="threadCn">分多少个线程来做</param>
/// <param name="processItemMethod">处理数据集合中单个数据使用的处理方法</param>
/// <param name="needWaitAll">是否需要等待所有线程执行完毕才返回，为true时会等待所有线程执行完毕，否则则是在有一个线程执行完毕就返回</param>
public static void DoAsync(IList dataCollection, int threadCn, DoGetObjTask processItemMethod, out Hashtable processResult)
{
DoAsyncPrivate(dataCollection, threadCn, null, processItemMethod, true, true, out processResult);
}
/// <summary>
/// 执行多线程操作任务
/// </summary>
/// <param name="dataCollection">多线程操作的数据集合</param>
/// <param name="threadCn">分多少个线程来做</param>
/// <param name="processItemMethod">处理数据集合中单个数据使用的处理方法</param>
/// <param name="needWaitAll">是否需要等待所有线程执行完毕才返回，为true时会等待所有线程执行完毕，否则则是在有一个线程执行完毕就返回</param>
public static void DoAsync(IList dataCollection, int threadCn, WaitCallback processItemMethod, bool needWaitAll)
{
Hashtable hash;
DoAsyncPrivate(dataCollection, threadCn, processItemMethod, null, needWaitAll, false, out hash);
}
private static void DoAsyncPrivate(IList dataCollection, int threadCn, WaitCallback processItemMethod, DoGetObjTask getObjMethod, bool needWaitAll, bool hasReturnValue, out Hashtable processResult)
{
if (dataCollection == null) throw new ArgumentNullException("dataCollection");
if (threadCn >= 64 || threadCn < 2)
{
throw new ArgumentOutOfRangeException("threadCn", "threadCn 参数必须在2和64之间");
}
if (threadCn > dataCollection.Count) threadCn = dataCollection.Count;
IList[] colls = new ArrayList[threadCn];
DataWithStateList dataWithStates = new DataWithStateList();
AutoResetEvent[] evts = new AutoResetEvent[threadCn];
for (int i = 0; i < threadCn; i++)
{
colls[i] = new ArrayList();
evts[i] = new AutoResetEvent(false);
}
for (int i = 0; i < dataCollection.Count; i++)
{
object obj = dataCollection[i];
int threadIndex = i % threadCn;
colls[threadIndex].Add(obj);
dataWithStates.Add(new DataWithState(obj, ProcessState.WaitForProcess));
}
AsyncContext context = AsyncContext.GetContext(threadCn, dataWithStates, needWaitAll, hasReturnValue, processItemMethod, getObjMethod);
for (int i = 0; i < threadCn; i++)
{
ThreadPool.QueueUserWorkItem(DoPrivate, new object[] {
colls[i],context,evts[i]
});
}
if (needWaitAll)
{
WaitHandle.WaitAll(evts);
}
else
{
WaitHandle.WaitAny(evts);
context.SetBreakSignal();
}
processResult = context.ProcessResult;
}
private class AsyncContext
{
static public AsyncContext GetContext(
int threadCn,
DataWithStateList dataWithStates,
bool needWaitAll,
bool hasReturnValue,
WaitCallback processItemMethod,
DoGetObjTask hasReturnValueMethod
)
{
AsyncContext context = new AsyncContext();
context.ThreadCount = threadCn;
context.DataWithStates = dataWithStates;
context.NeedWaitAll = needWaitAll;
if (hasReturnValue)
{
Hashtable processResult = Hashtable.Synchronized(new Hashtable());
context.ProcessResult = processResult;
context.HasReturnValueMethod = hasReturnValueMethod;
}
else
{
context.VoidMethod = processItemMethod;
}
context.HasReturnValue = hasReturnValue;
return context;
}
internal int ThreadCount;
internal DataWithStateList DataWithStates;
internal bool NeedWaitAll;
internal bool HasReturnValue;
internal WaitCallback VoidMethod;
internal DoGetObjTask HasReturnValueMethod;
private bool _breakSignal;
private Hashtable _processResult;
internal Hashtable ProcessResult
{
get { return _processResult; }
set { _processResult = value; }
}
internal void SetReturnValue(object obj, object result)
{
lock (_processResult.SyncRoot)
{
_processResult[obj] = result;
}
}
internal void SetBreakSignal()
{
if (NeedWaitAll) throw new NotSupportedException("设定为NeedWaitAll时不可设置BreakSignal");
_breakSignal = true;
}
internal bool NeedBreak
{
get
{
return !NeedWaitAll && _breakSignal;
}
}
internal void Exec(object obj)
{
if (HasReturnValue)
{
SetReturnValue(obj, HasReturnValueMethod(obj));
}
else
{
VoidMethod(obj);
}
DataWithStates.SetState(obj, ProcessState.Processed);
}
}
private enum ProcessState : byte
{
WaitForProcess = 0,
Processing = 1,
Processed = 2
}
private class DataWithStateList : List<DataWithState>
{
public void SetState(object obj, ProcessState state)
{
lock (((ICollection)this).SyncRoot)
{
DataWithState dws = this.Find(delegate(DataWithState i) { return Object.Equals(i.Data, obj); });
if (dws != null)
{
dws.State = state;
}
}
}
public ProcessState GetState(object obj)
{
lock (((ICollection)this).SyncRoot)
{
DataWithState dws = this.Find(delegate(DataWithState i) { return Object.Equals(i.Data, obj); });
return dws.State;
}
}
private int GetCount(ProcessState state)
{
List<DataWithState> datas = this.FindAll(delegate(DataWithState i) { return i.State == state; });
if (datas == null) return 0;
return datas.Count;
}
public int WaitForDataCount
{
get
{
return GetCount(ProcessState.WaitForProcess);
}
}
internal object GetWaitForObject()
{
lock (((ICollection)this).SyncRoot)
{
DataWithState dws = this.Find(delegate(DataWithState i) { return i.State == ProcessState.WaitForProcess; });
if (dws == null) return null;
dws.State = ProcessState.Processing;
return dws.Data;
}
}
internal bool IsWaitForData(object obj, bool setState)
{
lock (((ICollection)this).SyncRoot)
{
DataWithState dws = this.Find(delegate(DataWithState i) { return i.State == ProcessState.WaitForProcess; });
if (setState && dws != null) dws.State = ProcessState.Processing;
return dws != null;
}
}
}
private class DataWithState
{
public readonly object Data;
public ProcessState State;
public DataWithState(object data, ProcessState state)
{
Data = data;
State = state;
}
}
private static int _threadNo = 0;
private static void DoPrivate(object state)
{
object[] objs = state as object[];
IList datas = objs[0] as IList;
AsyncContext context = objs[1] as AsyncContext;
AutoResetEvent evt = objs[2] as AutoResetEvent;
DataWithStateList objStates = context.DataWithStates;
#if DEBUG
Thread.CurrentThread.Name = "Thread " + _threadNo;
Interlocked.Increment(ref _threadNo);
string threadName = Thread.CurrentThread.Name + "[" + Thread.CurrentThread.ManagedThreadId + "]";
Trace.WriteLine("线程ID:" + threadName);
#endif
if (datas != null)
{
for (int i = 0; i < datas.Count; i++)
{
if (context.NeedBreak)
{
#if DEBUG
Trace.WriteLine("线程" + threadName + "未执行完跳出");
#endif
break;
}
object obj = datas[i];
if (objStates.IsWaitForData(obj, true))
{
if (context.NeedBreak)
{
#if DEBUG
Trace.WriteLine("线程" + threadName + "未执行完跳出");
#endif
break;
}
context.Exec(obj);
#if DEBUG
Trace.WriteLine(string.Format("线程{0}处理{1}", threadName, obj));
#endif
}
}
}
if (context.NeedWaitAll)
{
//如果执行完当前进程的数据，还要查看剩下多少没有做，如果还剩下超过ThreadCount个没有做
while (objStates.WaitForDataCount > context.ThreadCount)
{
if (context.NeedBreak) break;
object obj = objStates.GetWaitForObject();
if (obj != null && objStates.IsWaitForData(obj, false))
{
if (context.NeedBreak)
{
#if DEBUG
Trace.WriteLine("线程" + threadName + "未执行完跳出");
#endif
break;
}
context.Exec(obj);
#if DEBUG
Trace.WriteLine(string.Format("线程{0}执行另一个进程的数据{1}", threadName, obj));
#endif
}
}
}
evt.Set();
}
}
}
如何使用AsyncHelper类，请看下面的测试代码：
using System;
using System.Collections.Generic;
using System.Text;
using System.Diagnostics;
using AppUtility;
using System.IO;
using System.Collections;
using System.Threading;
namespace ConsoleApplication2
{
class Program
{
static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
sw.Start();
/*
List<string> testFiles = new List<string>();
for (int i = 0; i < 100; i++)
{
testFiles.Add("D:\\test\\async\\file_" + i.ToString() + ".log");
}
AsyncHelper.DoAsync(testFiles, 10, WriteFile);
Console.WriteLine("异步写耗时"+sw.ElapsedMilliseconds + "ms");
*/
List<string> testFiles = new List<string>();
for (int i = 0; i < 200; i++)
{
testFiles.Add("D:\\test\\async\\file_" + i.ToString() + ".log");
}
Hashtable result;
AsyncHelper.DoAsync(testFiles, 20, WriteFileAndReturnRowCount,false,out result);
Console.WriteLine("异步写耗时" + sw.ElapsedMilliseconds + "ms");
Thread.Sleep(10);
if (result != null)
{
foreach (object key in result.Keys)
{
Console.WriteLine("{0}={1}", key,result[key]);
}
}
sw.Reset();
sw.Start();
for (int i = 0; i < 200; i++)
{
WriteFile("D:\\test\\sync\\file_" + i.ToString() + ".log");
}
Console.WriteLine("同步写耗时" + sw.ElapsedMilliseconds + "ms");
Console.Read();
}
static void WriteFile(object objFilePath)
{
string filePath = (string)objFilePath;
string dir = Path.GetDirectoryName(filePath);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
//Random r = new Random(DateTime.Now.Minute);
int rowCn = 10000;
using (StreamWriter writer = new StreamWriter(filePath, false, Encoding.Default))
{
for (int i = 0; i < rowCn; i++) writer.WriteLine(Guid.NewGuid());
}
}
static object WriteFileAndReturnRowCount(object objFilePath)
{
string filePath = (string)objFilePath;
string dir = Path.GetDirectoryName(filePath);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
//Random r = new Random(DateTime.Now.Minute);
int rowCn = 10000;
using (StreamWriter writer = new StreamWriter(filePath, false, Encoding.Default))
{
for (int i = 0; i < rowCn ; i++) writer.WriteLine(Guid.NewGuid());
}
return DateTime.Now.ToLongTimeString();
}
}
}
Sorry，代码太多，文字太少。发个牢骚，代码写完之后，再写思路是一件痛苦的事情！
