预热请求，多任务发送请求并获取请求结果保存到数据库中

class MyInit
    {
        ALiPayHelper aLiPayHelper = new ALiPayHelper();
        HttpClientRequest request = new HttpClientRequest();

        public void Init()
        {
            Mapper.Initialize(x => x.CreateMap<ALiPay, ALiPaySn>());
            Task.Run(() =>
            {
                while (true)
                {
                    AnalysisDataTimer_Elapsed(null);
                    Thread.Sleep(Helper.AnalysisHtmlTime);
                }
            });
            Task.Run(() =>
            {
                while (true)
                {
                    SendDataTimer_Elapsed(null);
                    Thread.Sleep(Helper.SendRquestTime);
                }
            });
            Task.Run(() =>
            {
                while (true)
                {
                    DeleteHtmlsTimer_Elapsed(null);
                    Thread.Sleep(12000);
                }                
            });
            ////解析定时器
            //var analysisDataTimer = new Timer(new TimerCallback(AnalysisDataTimer_Elapsed), null, 0, Helper.AnalysisHtmlTime);
            ////发送请求定时器
            //var sendDataTimer = new Timer(new TimerCallback(SendDataTimer_Elapsed), null, 0, Helper.SendRquestTime);
        }

        void DeleteHtmlsTimer_Elapsed(object sender)
        {
            try
            {
                using (var db = new ALiPayContext())
                {
                    db.Database.ExecuteSqlCommand("delete from alipayhtmls where IsProccessed=1");
                }
            }
            catch (Exception ex)
            {
                Logger.Default.Error($"删除已解析的数据报错{ex.Message}");
            }
        }
        void AnalysisDataTimer_Elapsed(object sender)
        {
            try
            {
                Logger.Default.Debug($"开始解析数据");
                DateTime nowTime = DateTime.Now;
                var dbWrite = new ALiPayContext();
                aLiPayHelper.AnalysisData(dbWrite, nowTime);
                Logger.Default.Debug($"完成解析数据");
            }
            catch (Exception ex)
            {
                Logger.Default.Error($"解析数据报错{ex.Message}");
            }
        }


        void SendDataTimer_Elapsed(object sender)
        {
            try
            {
                Logger.Default.Debug($"开始发送请求");
                DateTime nowTime = DateTime.Now;
                var dbWrite = new ALiPayContext();
                aLiPayHelper.SendData(dbWrite, request, nowTime);
                Logger.Default.Debug($"完成发送请求");
            }
            catch (Exception ex)
            {
                Logger.Default.Error($"发送请求报错{ex.Message}");
            }
        }
    }

class ALiPayHelper
    {
        #region 发送请求
        public void SendData(ALiPayContext dbWrite, HttpClientRequest request, DateTime nowTime)
        {
            var snList = dbWrite.ALiPaySn.Where(w => w.NextRequestTime < nowTime && !w.IsRequestSuccess).Take(200).ToList();
            dbWrite.Dispose();
            dbWrite = null;
            if (snList.Count > 0)
            {
                var taskList = new List<Task<ResultOut>>();
                foreach (var item in snList)
                {
                    taskList.Add(RequestData(request, Helper.RequestUri, item));
                }
                Task.WhenAll(taskList).ContinueWith(task =>
                {
                    if (task.IsFaulted)
                    {
                        Logger.Default.Error($"1,请求报错：{task.Exception.GetBaseException().Message}");
                    }
                    else
                    {
                        Logger.Default.Debug($"2,请求结果：{JsonConvert.SerializeObject(task.Result)}");
                        return ModifyRequestNum(task.Result);
                    }
                    return Task.FromResult(0);
                });
            }
        }
        /// <summary>
        /// 请求完成后将请求的结果信息修改到对应数据中
        /// </summary>
        /// <param name="list"></param>
        /// <returns></returns>
        Task<int> ModifyRequestNum(ResultOut[] list)
        {
            var paySnList = list.Select(w => w.Model.PaySn).ToList();
            if (paySnList.Count > 0)
            {
                using (var dbWrite = new ALiPayContext())
                {
                    var paySnModelList = dbWrite.ALiPaySn.Where(w => paySnList.Contains(w.PaySn)).ToList();
                    if (paySnModelList != null)
                    {
                        foreach (var paySnModel in paySnModelList)
                        {
                            var item = list.Where(w => w.Model.PaySn == paySnModel.PaySn).FirstOrDefault();
                            if (item != null)
                            {
                                paySnModel.IsRequestSuccess = item.IsSuccess;
                                paySnModel.RequestNum = paySnModel.RequestNum + 1;
                                paySnModel.ResponseStr = item.Result;
                                if (!item.IsSuccess)
                                {
                                    paySnModel.NextRequestTime = paySnModel.NextRequestTime.AddMinutes(paySnModel.RequestNum * 2);
                                }
                            }
                        }
                        var result = dbWrite.SaveChangesAsync();
                        return result;
                    }
                }
            }
            return Task.FromResult(0);
        }

        /// <summary>
        /// 发送请求,并返回请求结果信息
        /// </summary>
        /// <param name="request"></param>
        /// <param name="uri"></param>
        /// <param name="model"></param>
        /// <returns></returns>
        async Task<ResultOut> RequestData(HttpClientRequest request, string uri, ALiPaySn model)
        {
            var parameters = new List<KeyValuePair<string, string>>();
            parameters.Add(new KeyValuePair<string, string>("Money", model.Money.ToString("f2")));
            parameters.Add(new KeyValuePair<string, string>("Name", model.Name));
            parameters.Add(new KeyValuePair<string, string>("OtherSide", model.OtherSide));
            parameters.Add(new KeyValuePair<string, string>("PaySn", model.PaySn));
            parameters.Add(new KeyValuePair<string, string>("PayTime", model.PayTime));
            parameters.Add(new KeyValuePair<string, string>("Status", model.Status));
            var canshu = $"Money={model.Money.ToString("f2")}&Name={model.Name}&OtherSide={model.OtherSide}&PaySn={model.PaySn}&PayTime={model.PayTime}&Status={model.Status}&key={Helper.RequestMd5Key}";
            parameters.Add(new KeyValuePair<string, string>("sign", canshu.To32MD5()));
            Logger.Default.Debug($"uri：{uri},请求参数：{JsonConvert.SerializeObject(parameters)}");
            var postTask = request.PostAsync(uri, parameters);
            var timeOutTask = Task.Delay(10000);
            var task = await Task.WhenAny(postTask, timeOutTask);//返回任何一个完成的任务
            if (task == timeOutTask)//如果超时任务先完成了 就返回 很可惜 他没有完成！
            {
                Logger.Default.Debug($"uri：{uri},请求参数：{JsonConvert.SerializeObject(parameters)},结果：请求超时");
                return new ResultOut { IsSuccess = false, Msg = "请求超时", Model = model };
            }
            if (postTask.IsFaulted)
            {
                Logger.Default.Debug($"uri：{uri},请求参数：{JsonConvert.SerializeObject(parameters)},结果：请求错误.{task.Exception.GetBaseException().Message}");
                return new ResultOut() { IsSuccess = false, Msg = $"请求错误.{task.Exception.GetBaseException().Message}", Model = model };
            }
            //if (postTask.Result.Contains("success"))
            //{            
            Logger.Default.Debug($"uri：{uri},请求参数：{JsonConvert.SerializeObject(parameters)},结果：请求成功,{postTask.Result}");
            return new ResultOut { IsSuccess = true, Msg = "请求成功", Result = postTask.Result, Model = model };
            //}
            //return new ResultOut { IsSuccess = false, Msg = "请求成功", Result = postTask.Result, Model = model };
        }
        #endregion

        #region 解析数据并保存        
        public void AnalysisData(ALiPayContext dbWrite, DateTime nowTime)
        {
            var list = dbWrite.AlipayHtmls.Where(w => !w.IsProccessed).OrderByDescending(o => o.CreateTime).Take(200).ToList();
            if (list.Count > 0)
            {
                var aLiPayList = Send(list);
                aLiPayList.RemoveAll(w => dbWrite.ALiPaySn.Any(t => t.PaySn == w.PaySn));

                if (aLiPayList.Count > 0)
                {
                    foreach (var item in aLiPayList)
                    {
                        var dto = Mapper.Map<ALiPaySn>(item);
                        dto.Id = Guid.NewGuid();
                        dto.NextRequestTime = nowTime;
                        dto.CreateTime = nowTime;
                        dto.RequestNum = 0;
                        dbWrite.ALiPaySn.Add(dto);
                    }
                }
                foreach (var item in list)
                {
                    item.IsProccessed = true;
                }
                var taskHtml = dbWrite.SaveChangesAsync();
                taskHtml.ContinueWith(task =>
                {
                    task.Wait();
                    dbWrite.Dispose();
                    dbWrite = null;
                    if (taskHtml.IsFaulted)
                    {
                        StringBuilder exSb = new StringBuilder();
                        var exList = taskHtml.Exception.InnerExceptions;
                        foreach (var ex in exList)
                        {
                            if (ex is System.Data.Entity.Infrastructure.DbUpdateException)
                            {
                                exSb.Append(ex.InnerException.InnerException.Message);
                            }
                            else if (ex is DbEntityValidationException)
                            {
                                var vex = ex as DbEntityValidationException;
                                foreach (var entityItem in vex.EntityValidationErrors)
                                {
                                    foreach (var err in entityItem.ValidationErrors)
                                    {
                                        exSb.Append(err.ErrorMessage);
                                    }
                                }
                            }
                            else
                            {
                                exSb.Append(ex.Message);
                            }
                        }
                        Logger.Default.Error(exSb.ToString());
                    }
                    else
                    {
                        Logger.Default.Debug($"解析后添加数据,受影响的行数{taskHtml.Result.ToString()}条");
                    }
                });
            }
        }

        #region 解析JSON
        List<ALiPay> Send(List<AlipayHtmls> list)
        {
            var aliPayList = new List<ALiPay>();
            Newtonsoft.Json.Linq.JObject json, result;
            Newtonsoft.Json.Linq.JArray detailArry;
            string status;
            foreach (var item in list)
            {
                json = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(item.Html);
                status = json["status"].ToString();
                if (status == "succeed")
                {
                    result = (Newtonsoft.Json.Linq.JObject)json["result"];
                    detailArry = (Newtonsoft.Json.Linq.JArray)result["detail"];
                    foreach (var detailItem in detailArry)
                    {
                        var model = new ALiPay
                        {
                            MoneyStr = detailItem["totalAmount"].ToString(),
                            Name = detailItem["goodsTitle"].ToString(),
                            OtherSide = detailItem["consumerLoginId"].ToString(),
                            PaySn = detailItem["tradeNo"].ToString(),
                            PayTime = detailItem["gmtCreate"].ToString(),
                            Status = detailItem["tradeStatus"].ToString()
                        };
                        if (!aliPayList.Any(w => w.PaySn == model.PaySn))
                        {
                            aliPayList.Add(model);
                        }
                    }
                }
            }
            return aliPayList;
        }
        #endregion

        #region 解析html
        //List<ALiPay> Send(List<AlipayHtmls> list)
        //{
        //    var doc = new HtmlDocument();
        //    var aliPayList = new List<ALiPay>();
        //    foreach (var item in list)
        //    {
        //        doc.LoadHtml(item.Html);
        //        var tradeRecordsIndexTabEle = doc.GetElementbyId("tradeRecordsIndex");
        //        var trEles = tradeRecordsIndexTabEle.SelectNodes("//tbody/tr");
        //        foreach (var tr in trEles)
        //        {
        //            var model = new ALiPay();
        //            var memoEle = tr.SelectSingleNode("td[@class='memo']/a");
        //            if (memoEle == null)
        //            {
        //                var nameEle = tr.SelectSingleNode("td[@class='name']/p/a");
        //                if (nameEle == null)
        //                {
        //                    nameEle = tr.SelectSingleNode("td[@class='action']/select/option[@class='split']");
        //                    GetOrderOn(nameEle.GetAttributeValue("data-link", string.Empty).RemoveEmpty(), model);

        //                    nameEle = tr.SelectSingleNode("td[@class='name']");
        //                    model.Name = nameEle.InnerText.RemoveEmpty();
        //                }
        //                else
        //                {
        //                    GetOrderOn(nameEle.GetAttributeValue("href", string.Empty), model);
        //                    model.Name = nameEle.InnerText.RemoveEmpty();
        //                }
        //            }
        //            else
        //            {
        //                var nameEle = tr.SelectSingleNode("td[@class='name']");
        //                model.Name = nameEle.InnerText.RemoveEmpty();
        //                GetOrderOn(memoEle.GetAttributeValue("href", string.Empty), model);
        //            }
        //            if (aliPayList.Any(w => w.PaySn == model.PaySn))
        //            {
        //                continue;
        //            }
        //            if (model.Name.StartsWith("商品-"))
        //            {
        //                model.Name = model.Name.Substring(3);
        //            }
        //            var otherEle = tr.SelectSingleNode("td[@class='other']");
        //            model.OtherSide = otherEle == null ? string.Empty : otherEle.InnerText.RemoveEmpty();

        //            var p0Ele = tr.SelectSingleNode("td[@class='name']/p[@class='consume-title']");
        //            var payName = p0Ele.InnerText.RemoveEmpty();

        //            var amountEle = tr.SelectSingleNode("td[@class='amount']");
        //            model.MoneyStr = amountEle.InnerText.RemoveEmpty();

        //            var statusEle = tr.SelectSingleNode("td[@class='status']/p");
        //            model.Status = statusEle.InnerText.RemoveEmpty();
        //            aliPayList.Add(model);
        //        }
        //    }
        //    return aliPayList;
        //}
        //static Regex regEditMemo = new Regex("/record/editMemo.htm?", RegexOptions.IgnoreCase);
        //static Regex regSimpleDetail = new Regex("/record/detail/simpleDetail.htm?", RegexOptions.IgnoreCase);
        //static Regex regBizInNo = new Regex(@"bizInNo=(\d+)", RegexOptions.IgnoreCase);
        //static Regex regGmtBizCreate = new Regex(@"gmtBizCreate=(\d+)", RegexOptions.IgnoreCase);
        //static Regex regCreateDate = new Regex(@"createDate=&#9;&#9;&#9;(\d+)", RegexOptions.IgnoreCase);
        //static Regex regCreateDate1 = new Regex(@"createDate=(\d+)", RegexOptions.IgnoreCase);
        //static void GetOrderOn(string url, ALiPay model)
        //{
        //    model.PaySn = regBizInNo.Match(url).Groups[1].Value;
        //    if (regEditMemo.IsMatch(url))
        //    {
        //        model.PayTime = regCreateDate.Match(url).Groups[1].Value;
        //        if (string.IsNullOrWhiteSpace(model.PayTime))
        //        {
        //            model.PayTime = regCreateDate1.Match(url).Groups[1].Value;
        //        }
        //    }
        //    else if (regSimpleDetail.IsMatch(url))
        //    {
        //        model.PayTime = regGmtBizCreate.Match(url).Groups[1].Value;
        //    }
        //}
        #endregion

        #endregion
    }

 

 /// <summary>
        /// 移除空字符包括换行,制表符,空格
        /// </summary>
        /// <param name="str"></param>
        /// <returns></returns>
        public static string RemoveEmpty(this string str)
        {
            if (string.IsNullOrEmpty(str))
            {
                return string.Empty;
            }
            return str.Replace("\n", "").Replace(" ", "").Replace("\t", "").Replace("\r", "");
        }

        /// <summary>
        /// 转换为数字
        /// </summary>
        /// <param name="str"></param>
        /// <returns></returns>
        public static int ToMyInt32(this string str, int defaultVal = 0)
        {
            if (string.IsNullOrEmpty(str))
            {
                return defaultVal;
            }
            int.TryParse(str, out defaultVal);
            return defaultVal;
        }

        static MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider();
        /// <summary>
        /// 32位MD5加密
        /// </summary>
        /// <param name="val"></param>
        /// <returns></returns>
        public static string To32MD5(this string val)
        {
            var t2 = BitConverter.ToString(md5.ComputeHash(Encoding.UTF8.GetBytes(val)));
            t2 = t2.Replace("-", "");
            return t2;
        }

        /// <summary>
        /// 获取时间戳（取到毫秒得到的时间戳就是13位，只取到秒的话时间戳就是10位）
        /// </summary>
        /// <param name="dt"></param>
        /// <param name="bflag"></param>
        /// <returns></returns>
        public static long MyDateToLong(this DateTime dt, bool bflag = false)
        {
            TimeSpan ts = dt - new DateTime(1970, 1, 1, 0, 0, 0, 0);            
            if (bflag)
            { 
                return Convert.ToInt64(ts.TotalSeconds);
            }
            return Convert.ToInt64(ts.TotalMilliseconds);
        }
    }

    /// <summary>
    /// 请求帮助类
    /// </summary>
    public class HttpClientRequest
    {
        private readonly HttpClient _httpClient;
        public HttpClientRequest()
        {
            _httpClient = new HttpClient() { BaseAddress = new Uri(Helper.RequestUrl) };

            //帮HttpClient热身
            _httpClient.SendAsync(new HttpRequestMessage
            {
                Method = new HttpMethod("HEAD"),
                RequestUri = new Uri(Helper.RequestUrl)
            }).Result.EnsureSuccessStatusCode();
        }
        /// <summary>
        /// POST请求
        /// </summary>
        /// <param name="uri"></param>
        /// <param name="parameters"></param>
        /// <returns></returns>
        public async Task<string> PostAsync(string uri, List<KeyValuePair<string, string>> parameters)
        {
            var response = await _httpClient.PostAsync(uri, new FormUrlEncodedContent(parameters));

            return await response.Content.ReadAsStringAsync();
        }
    }

    public static class ListEx
    {
        #region 将datatable数据转化成list

        /// <summary>
        /// 将datatable数据转化成list
        /// </summary>
        /// <typeparam name="T">泛型T</typeparam>
        /// <param name="dt">对应的datatable数据表</param>
        /// <returns>返回结果的数据集</returns>
        public static List<T> ToLists<T>(this DataTable dt) where T : class, new()
        {
            Type t = typeof(T);
            PropertyInfo[] propertys = t.GetProperties();
            List<T> lst = new List<T>();
            string typeName = string.Empty;

            foreach (DataRow dr in dt.Rows)
            {
                T entity = new T();
                foreach (PropertyInfo pi in propertys)
                {
                    typeName = pi.Name;
                    if (dt.Columns.Contains(typeName))
                    {
                        if (!pi.CanWrite) continue;
                        object value = dr[typeName];
                        if (value == DBNull.Value) continue;
                        if (pi.PropertyType == typeof(string))
                        {
                            pi.SetValue(entity, value.ToString(), null);
                        }
                        else if (pi.PropertyType == typeof(int) || pi.PropertyType == typeof(int?))
                        {
                            pi.SetValue(entity, int.Parse(value.ToString()), null);
                        }
                        else if (pi.PropertyType == typeof(DateTime?) || pi.PropertyType == typeof(DateTime))
                        {
                            pi.SetValue(entity, DateTime.Parse(value.ToString()), null);
                        }
                        else if (pi.PropertyType == typeof(float))
                        {
                            pi.SetValue(entity, float.Parse(value.ToString()), null);
                        }
                        else if (pi.PropertyType == typeof(double))
                        {
                            pi.SetValue(entity, double.Parse(value.ToString()), null);
                        }
                        else
                        {
                            pi.SetValue(entity, value, null);
                        }
                    }
                }
                lst.Add(entity);
            }
            return lst;

            //调用
            //List<People> p = dt.ToList<People>();
        }

        #endregion

        #region  将list数据转化成datatable
        public static DataTable ToDataTable(this IList list, params string[] propertyName)
        {
            List<string> propertyNameList = new List<string>();
            if (propertyName != null)
            {
                propertyNameList.AddRange(propertyName);
            }
            DataTable result = new DataTable();
            if (list.Count > 0)
            {
                PropertyInfo[] propertys = list[0].GetType().GetProperties();
                foreach (PropertyInfo pi in propertys)
                {
                    if (propertyNameList.Count == 0)
                    {
                        result.Columns.Add(pi.Name, pi.PropertyType);
                    }
                    else
                    {
                        if (propertyNameList.Contains(pi.Name))
                            result.Columns.Add(pi.Name, pi.PropertyType);
                    }
                }
                for (int i = 0; i < list.Count; i++)
                {
                    ArrayList tempList = new ArrayList();
                    foreach (PropertyInfo pi in propertys)
                    {
                        if (propertyNameList.Count == 0)
                        {
                            object obj = pi.GetValue(list[i], null);
                            tempList.Add(obj);
                        }
                        else
                        {
                            if (propertyNameList.Contains(pi.Name))
                            {
                                object obj = pi.GetValue(list[i], null);
                                tempList.Add(obj);
                            }
                        }
                    }
                    object[] array = tempList.ToArray();
                    result.LoadDataRow(array, true);
                }
            }
            return result;
        }
        #endregion
    }