C# 微信生活助手空气质量天气预报等效果展示数据抓取（二）

此文主要是中国天气网和中国环境监测总站的数据抓取打算开放全部数据抓取源代码已在服务器上稳定运行半个月

常量

 1 /// <summary>
 2         /// 环保部抓取数据链接
 3         /// </summary>
 4         public const string HBUrl = "http://datacenter.mep.gov.cn/report/air_daily/airDairyCityHour.jsp";
 5         /// <summary>
 6         /// 环境检测总站
 7         /// </summary>
 8         public const string HJJCZZUrl = "http://113.108.142.147:20035/emcpublish/ClientBin/Env-CnemcPublish-RiaServices-EnvCnemcPublishDomainService.svc/binary";
 9         /// <summary>
10         /// 获取所有站点AQI数据
11         /// </summary>
12         public const string AQIDataUrl = HJJCZZUrl + "/GetAQIDataPublishLives";
13         /// <summary>
14         /// 获取所有城市AQI数据
15         /// </summary>
16         public const string CityDataUrl = HJJCZZUrl + "/GetCityAQIPublishLives";
17         /// <summary>
18         /// 中国天气网
19         /// </summary>
20         public const string WeatherUrl = "http://www.weather.com.cn";
21         /// <summary>
22         /// 中国天气网7天天气预报
23         /// </summary>
24         //public const string Weather7DUrl = WeatherUrl + "/weather/{0}.shtml";
25         public const string Weather7DUrl = "http://home.cw.uitv.com.cn/Weather.ashx?service=2&cityid={0}";
26         /// <summary>
27         /// 中国天气网实况天气预报
28         /// </summary>
29         //public const string WeatherSKUrl = "http://data.weather.com.cn/sk/{0}.html";
30         public const string WeatherSKUrl = "http://d1.weather.com.cn/sk_2d/{0}.html";
31         public const string WeatherSKDetailUrl = "http://flash.weather.com.cn/wmaps/xml/{0}.xml";
32 
33 
34         public static readonly string[] ChinaProvinces = new string[] { "heilongjiang", "jilin", "liaoning", "hainan", "neimenggu", "xinjiang", "xizang", "qinghai", "ningxia", "gansu", "hebei", "henan", "hubei", "hunan", "shandong", "jiangsu", "anhui", "shanxi", "sanxi", "sichuan", "yunnan", "guizhou", "zhejiang", "fujian", "jiangxi", "guangdong", "guangxi", "beijing", "tianjin", "shanghai", "chongqing", "xianggang", "aomen", "taiwan", "xisha", "nanshadao", "diaoyudao" };
35 
36         /// <summary>
37         /// 风向
38         /// </summary>
39         public static readonly string[] WindDirections = new string[] { "无持续风向", "东北风", "东风", "东南风", "南风", "西南风", "西风", "西北风", "北风", "旋转风" };
40         /// <summary>
41         /// 风力
42         /// </summary>
43         public static readonly string[] WindForces = new string[] { "微风", "3-4级", "4-5级", "5-6级", "6-7级", "7-8级", "8-9级", "9-10级", "10-11级", "11-12级" };
44         /// <summary>
45         /// 天气
46         /// </summary>
47         public static readonly string[] Weathers = new string[] { "晴", "多云", "阴", "阵雨", "雷阵雨", "冰雹", "雨夹雪", "小雨", "中雨", "大雨", "暴雨", "大暴雨", "特大暴雨", "阵雪", "小雪", "中雪", "大雪", "暴雪", "雾", "冻雨", "沙尘暴", "小到中雨", "中到大雨", "大到暴雨", "暴雨到大暴雨", "大暴雨到特大暴雨", "小到中雪", "中到大雪", "大到暴雪", "浮尘", "扬沙", "强沙尘暴", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "霾" };
48         /// <summary>
49         /// 质数代号
50         /// </summary>
51         public static readonly string[] Indexs = new string[] { "ag", "cl", "lk", "yh", "uv", "ls", "mf", "pj", "tr", "yd", "hc", "ac", "ct", "gj", "ys", "nl", "pk", "zs", "xc", "co", "dy", "fs", "gm", "jt", "xq", "pl", "pp" };
52         /// <summary>
53         /// 质数名称
54         /// </summary>
55         public static readonly string[] IndexNames = new string[] { "息斯敏过敏指数", "晨练指数", "路况指数", "约会指数", "紫外线强度指数", "晾晒指数", "美发指数", "啤酒指数", "旅游指数", "运动指数", "划船指数", "空调开启指数", "穿衣指数", "逛街指数", "雨伞指数", "夜生活指数", "放风筝指数", "中暑指数", "洗车指数", "舒适度指数", "钓鱼指数", "防晒指数", "感冒指数", "交通指数", "心情指数", "空气污染扩散条件指数", "化妆指数" };

View Code

中国天气网没什么要说的直接抓取就可以了代码如下

7天天气预报一天发布3次

 1 foreach (int areaId in weatherAreaIds)
 2             {
 3                 if (weather7DLastGrabTime[areaId] < DateTime.Today || ((weather7DLastGrabTime[areaId] == DateTime.Today.AddHours(8) && DateTime.Now.Hour > 10) ||
 4                    (weather7DLastGrabTime[areaId] == DateTime.Today.AddHours(11) && DateTime.Now.Hour > 17) ||
 5                    (weather7DLastGrabTime[areaId] == DateTime.Today.AddDays(-1).AddHours(18) && DateTime.Now.Hour > 7)))
 6                 {
 7                     string res = GetRequest(string.Format(Constant.Weather7DUrl, areaId), "isexist=1");
 8                     try
 9                     {
10                         JsonForecast jsonForecast = JsonConvert.DeserializeObject<JsonForecast>(res);
11                         Forecast7D forecast7D = Mapper.Map<Qtyb, Forecast7D>(jsonForecast.qtyb);
12                         forecast7D.City = jsonForecast.city;
13                         forecast7D.CityId = areaId;
14                         forecast7D.Time = jsonForecast.ftime;
15                         if (jsonForecast.ftime == weather7DLastGrabTime[areaId])
16                         {
17                             Thread.Sleep(1000);
18                             continue;
19                         }
20                         lock (aqiEntities)
21                         {
22                             weather7DLastGrabTime[areaId] = forecast7D.Time;
23                             aqiEntities.GrabDataConfig.FirstOrDefault(p => p.Key == areaId.ToString() && p.Type == 5).Time = forecast7D.Time;
24                             aqiEntities.Forecast7D.Add(forecast7D);
25                             aqiEntities.SaveChanges();
26                             Log(string.Format("【天气预报】 {0} 发布时间：{1} 获取时间：{2}", forecast7D.City, forecast7D.Time, DateTime.Now));
27                         }
28                     }
29                     catch (Exception)
30                     {
31                     }
32                 }
33             }

View Code

天气实况

 1 string res = GetRequest(string.Format(Constant.WeatherSKDetailUrl, area), Constant.WeatherUrl, "").Replace(area, "JsonSKDetail");
 2             using (StreamReader streamReader = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(res)), Encoding.UTF8))
 3             {
 4                 try
 5                 {
 6                     JsonSKDetail jsonSKDetail = (JsonSKDetail)xmlSerializer.Deserialize(streamReader);
 7                     foreach (JsonSKDetailCity jsonSKDetailCity in jsonSKDetail.city)
 8                     {
 9                         WeatherSKDetail weatherSKDetail = Mapper.Map<JsonSKDetailCity, WeatherSKDetail>(jsonSKDetailCity);
10                         weatherSKDetail.dn = jsonSKDetail.dn;
11                         if (!weatherSKDetails.Exists(p => p.url == weatherSKDetail.url) && weatherSKDetailLastGrabTime[weatherSKDetail.url] != weatherSKDetail.time)
12                         {
13                             weatherSKDetails.Add(weatherSKDetail);
14                             weatherSKDetailLastGrabTime[weatherSKDetail.url] = weatherSKDetail.time;
15                             lock (aqiEntities)
16                             {
17                                 aqiEntities.GrabDataConfig.FirstOrDefault(p => p.Type == 3 && p.Key == weatherSKDetail.url.ToString()).Time = weatherSKDetail.time;
18                                 aqiEntities.SaveChanges();
19                             }
20                         }
21                         if (!string.IsNullOrEmpty(jsonSKDetailCity.pyName))
22                         {
23                             GetSKDetail(jsonSKDetailCity.pyName);
24                         }
25                     }
26                 }
27                 catch (Exception ex)
28                 {
29                     string s = ex.Message;
30                 }
31             }

View Code

中国环境检测总站

通过抓包发现用的是wcf 返回的是二进制xml文档需要用到一个 https://github.com/waf/WCF-Binary-Message-Inspector 把数据解析出来同时他也是fiddler的一个插件可以在里面直接看出

 1 private void GetCityAQI()
 2         {
 3             bool isWriteFile = false;
 4             if (DateTime.Now >= lastGrabCityAQITime.AddHours(1))
 5             {
 6                 #region 获取数据
 7                 while (true)
 8                 {
 9                     try
10                     {
11                         XmlElement xmlElement = GetXmlElement(Constant.CityDataUrl);
12                         XmlNode xn = xmlElement.LastChild.LastChild;
13                         XmlNodeList xmlNodeList = xn.ChildNodes;
14                         DateTime TimePoint = DateTime.Now;
15                         if (xmlNodeList.Count > 0)
16                         {
17                             XmlElement xe = (XmlElement)xmlNodeList[0];
18                             TimePoint = DateTime.Parse(xe.GetElementsByTagName("b:TimePoint")[0].InnerText);
19                             if (TimePoint == lastGrabCityAQITime)
20                             {
21                                 isWriteFile = false;
22                             }
23                             else
24                             {
25                                 lastGrabCityAQITime = TimePoint;
26                                 isWriteFile = true;
27                             }
28                         }
29                         #region 数据操作
30                         if (isWriteFile)
31                         {
32                             foreach (XmlElement xe in xmlNodeList)
33                             {
34                                 string Area = xe.GetElementsByTagName("b:Area")[0].InnerText;
35                                 string PrimaryPollutant = xe.GetElementsByTagName("b:PrimaryPollutant")[0].InnerText;
36                                 if (PrimaryPollutant == "—")
37                                     PrimaryPollutant = null;
38                                 int? AQI = xe.GetElementsByTagName("b:AQI")[0].InnerText.ToNullable<int>();
39                                 int? SO2 = xe.GetElementsByTagName("b:SO2")[0].InnerText.ToNullable<int>();
40                                 int? PM2_5 = xe.GetElementsByTagName("b:PM2_5")[0].InnerText.ToNullable<int>();
41                                 int? PM10 = xe.GetElementsByTagName("b:PM10")[0].InnerText.ToNullable<int>();
42                                 int? O3 = xe.GetElementsByTagName("b:O3")[0].InnerText.ToNullable<int>();
43                                 int? NO2 = xe.GetElementsByTagName("b:NO2")[0].InnerText.ToNullable<int>();
44                                 float? CO = xe.GetElementsByTagName("b:CO")[0].InnerText.ToNullable<float>();
45                                 int CityCode = int.Parse(xe.GetElementsByTagName("b:CityCode")[0].InnerText);
46                                 var model = new CityAQI()
47                                 {
48                                     CityCode = CityCode,
49                                     Area = Area,
50                                     O3 = O3,
51                                     CO = CO,
52                                     TimePoint = TimePoint,
53                                     AQI = AQI,
54                                     PrimaryPollutant = PrimaryPollutant,
55                                     NO2 = NO2,
56                                     PM10 = PM10,
57                                     PM2_5 = PM2_5,
58                                     SO2 = SO2
59                                 };
60                                 cityAQIs.Add(model);
61                             }
62                             lock (aqiEntities)
63                             {
64                                 aqiEntities.CityAQI.AddRange(cityAQIs);
65                                 aqiEntities.GrabDataConfig.FirstOrDefault(p => p.Key == "CityAQI").Time = lastGrabCityAQITime;
66                                 aqiEntities.SaveChanges();
67                                 cityAQIs.Clear();
68                             }
69                             lb_log.Invoke((EventHandler)delegate
70                             {
71                                 if (lb_log.Items.Count > 200)
72                                 {
73                                     lb_log.Items.Clear();
74                                 }
75                                 lb_log.Items.Add("中国环境检测总站 城市" + " " + TimePoint.ToString("yyyy-MM-dd HH:mm:ss") + "数据 " + DateTime.Now);
76                                 lb_log.SelectedIndex = lb_log.Items.Count - 1;
77                             });
78                             Thread.Sleep(60 * 1000);
79                         }
80                         #endregion
81                         break;
82                     }
83                     catch (Exception)
84                     {
85                         Thread.Sleep(60 * 1000);
86                     }
87                 }
88                 #endregion
89             }
90         }

View Code

关注微信公众帐号获取最新更新微信号kxshzsQQ:519872449 QQ群：77877965

posted @ 2015-04-18 10:27 hengkx 阅读(2288) 评论(9) 编辑收藏举报

会员力量，点亮园子希望

刷新页面返回顶部

开心

C# 微信生活助手空气质量天气预报等效果展示数据抓取（二）

公告

开心

C# 微信 生活助手 空气质量 天气预报等 效果展示 数据抓取 （二）

公告

C# 微信生活助手空气质量天气预报等效果展示数据抓取（二）