c# HttpWebRequest与HttpWebResponse
2012-09-02 23:17 yezhi 阅读(510) 评论(0) 收藏 举报http://www.cnblogs.com/sufei/archive/2011/05/22/2053642.html
部分代码如下到我博客可以是看全部和例子
/// <summary> /// 类说明：HttpHelps类，用来实现Http访问，Post或者Get方式的，直接访问，带Cookie的，带证书的等方式，可以设置代理 /// 编码日期：2011-08-20 /// 编 码 人：苏飞 /// 联系方式：361983679 Blogs：http://www.cckan.net /// 修改日期：2012-08-31 /// </summary> using System; using System.Collections.Generic; using System.Text; using System.Net; using System.IO; using System.Text.RegularExpressions; using System.IO.Compression; namespace FunctionServices { public class httphelper { #region 预定义方法或者变更 //默认的编码 public Encoding encoding = Encoding.Default; //HttpWebRequest对象用来发起请求 public HttpWebRequest request = null; //获取影响流的数据对象 private HttpWebResponse response = null; public string cookie = ""; public Boolean isToLower = true; //读取流的对象 private StreamReader reader = null; //需要返回的数据对象 private string returnData = "String Error"; /// <summary> /// 根据相传入的数据，得到相应页面数据 /// </summary> /// <param name="strPostdata">传入的数据Post方式,get方式传NUll或者空字符串都可以</param> /// <returns>string类型的响应数据</returns> private string GetHttpRequestData(string strPostdata) { try { //支持跳转页面，查询结果将是跳转后的页面 request.AllowAutoRedirect = true; //验证在得到结果时是否有传入数据 if (!string.IsNullOrEmpty(strPostdata) && request.Method.Trim().ToLower().Contains("post")) { byte[] buffer = encoding.GetBytes(strPostdata); request.ContentLength = buffer.Length; request.GetRequestStream().Write(buffer, 0, buffer.Length); } ////最大连接数 //request.ServicePoint.ConnectionLimit = 1024; #region 得到请求的response using (response = (HttpWebResponse)request.GetResponse()) { try { cookie = response.Headers["set-cookie"].ToString(); } catch (Exception) { } //从这里开始我们要无视编码了 if (encoding == null) { MemoryStream _stream = new MemoryStream(); if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase)) { //开始读取流并设置编码方式 //new GZipStream(response.GetResponseStream(), CompressionMode.Decompress).CopyTo(_stream, 10240); //.net4.0以下写法 _stream = GetMemoryStream(response.GetResponseStream()); } else { //response.GetResponseStream().CopyTo(_stream, 10240); // .net4.0以下写法 _stream = GetMemoryStream(response.GetResponseStream()); } byte[] RawResponse = _stream.ToArray(); string temp = Encoding.Default.GetString(RawResponse, 0, RawResponse.Length); //<meta(.*?)charset([\s]?)=[^>](.*?)> Match meta = Regex.Match(temp, "<meta([^<]*)charset=([^<]*)[\"']", RegexOptions.IgnoreCase | RegexOptions.Multiline); string charter = (meta.Groups.Count > 2) ? meta.Groups[2].Value : string.Empty; charter = charter.Replace("\"", string.Empty).Replace("'", string.Empty).Replace(";", string.Empty); if (charter.Length > 0) { charter = charter.ToLower().Replace("iso-8859-1", "gbk"); encoding = Encoding.GetEncoding(charter); } else { if (response.CharacterSet.ToLower().Trim() == "iso-8859-1") { encoding = Encoding.GetEncoding("gbk"); } else { if (string.IsNullOrEmpty(response.CharacterSet.Trim())) { encoding = Encoding.UTF8; } else { encoding = Encoding.GetEncoding(response.CharacterSet); } } } returnData = encoding.GetString(RawResponse); } else { if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase)) { //开始读取流并设置编码方式 using (reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress), encoding)) { returnData = reader.ReadToEnd(); } } else { //开始读取流并设置编码方式 using (reader = new StreamReader(response.GetResponseStream(), encoding)) { returnData = reader.ReadToEnd(); } } } } #endregion } catch (WebException ex) { //这里是在发生异常时返回的错误信息 returnData = "String Error"; response = (HttpWebResponse)ex.Response; } if (isToLower) { returnData = returnData.ToLower(); } return returnData; } /// <summary> /// 4.0以下.net版本取水运 /// </summary> /// <param name="streamResponse"></param> private static MemoryStream GetMemoryStream(Stream streamResponse) { MemoryStream _stream = new MemoryStream(); int Length = 256; Byte[] buffer = new Byte[Length]; int bytesRead = streamResponse.Read(buffer, 0, Length); // write the required bytes while (bytesRead > 0) { _stream.Write(buffer, 0, bytesRead); bytesRead = streamResponse.Read(buffer, 0, Length); } return _stream; } /// <summary> /// 为请求准备参数 /// </summary> /// <param name="_URL">请求的URL地址</param> /// <param name="_Method">请求方式Get或者Post</param> /// <param name="_Accept">Accept</param> /// <param name="_ContentType">ContentType返回类型</param> /// <param name="_UserAgent">UserAgent客户端的访问类型，包括浏览器版本和操作系统信息</param> /// <param name="_Encoding">读取数据时的编码方式</param> private void SetRequest(string _URL, string _Method, string _Accept, string _ContentType, string _UserAgent, Encoding _Encoding) { //初始化对像，并设置请求的URL地址 request = (HttpWebRequest)WebRequest.Create(GetUrl(_URL)); //请求方式Get或者Post request.Method = _Method; //Accept request.Accept = _Accept; //ContentType返回类型 request.ContentType = _ContentType; //UserAgent客户端的访问类型，包括浏览器版本和操作系统信息 request.UserAgent = _UserAgent; //读取数据时的编码方式 encoding = _Encoding; } /// <summary> /// 设置当前访问使用的代理 /// </summary> /// <param name="userName">代理 服务器用户名</param> /// <param name="passWord">代理 服务器密码</param> /// <param name="ip">代理 服务器地址</param> public void SetWebProxy(string userName, string passWord, string ip) { //设置代理服务器 WebProxy myProxy = new WebProxy(ip, false); //建议连接 myProxy.Credentials = new NetworkCredential(userName, passWord); //给当前请求对象 request.Proxy = myProxy; //设置安全凭证 request.Credentials = CredentialCache.DefaultNetworkCredentials; } #endregion #region 普通类型 /// <summary> /// 传入一个正确或不正确的URl，返回正确的URL /// </summary> /// <param name="URL">url</param> /// <returns> /// </returns> public static string GetUrl(string URL) { if (!(URL.Contains("http://") || URL.Contains("https://"))) { URL = "http://" + URL; } return URL; } /// <summary> /// 采用https协议GET|POST方式访问网络,根据传入的URl地址，得到响应的数据字符串。 /// </summary> /// <param name="_URL"></param> /// <param name="_Method">请求方式Get或者Post</param> /// <param name="_Accept">Accept</param> /// <param name="_ContentType">ContentType返回类型</param> /// <param name="_UserAgent">UserAgent客户端的访问类型，包括浏览器版本和操作系统信息</param> /// <param name="_Encoding">读取数据时的编码方式</param> /// <param name="_Postdata">只有_Method为Post方式时才需要传入值</param> /// <returns>返回Html源代码</returns> public string GetHtml(string _URL, string _Method, string _Accept, string _ContentType, string _UserAgent, Encoding _Encoding, string _Postdata) { //准备参数 SetRequest(_URL, _Method, _Accept, _ContentType, _UserAgent, _Encoding); //调用专门读取数据的类 return GetHttpRequestData(_Postdata); } ///<summary> ///采用https协议GET方式访问网络,根据传入的URl地址，得到响应的数据字符串。 ///</summary> ///<param name="URL">url地址</param> ///<param name="objencoding">编码方式例如：System.Text.Encoding.UTF8;</param> ///<returns>String类型的数据</returns> public string GetHtml(string URL, Encoding objencoding) { //准备参数 SetRequest(URL, "GET", "text/html, application/xhtml+xml, */*", "text/html", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", objencoding); //调用专门读取数据的类 return GetHttpRequestData(""); } ///<summary> ///采用https协议GET方式访问网络,根据传入的URl地址，得到响应的数据字符串。 ///</summary> ///<param name="URL">url地址</param> ///<param name="objencoding">编码方式例如：System.Text.Encoding.UTF8;</param> ///<param name="stgrcookie">Cookie字符串</param> ///<returns>String类型的数据</returns> public string GetHtml(string URL, Encoding objencoding, string stgrcookie) { //准备参数 SetRequest(URL, "GET", "text/html, application/xhtml+xml, */*", "text/html", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", objencoding); request.Headers[HttpRequestHeader.Cookie] = stgrcookie; //调用专门读取数据的类 return GetHttpRequestData(""); } ///<summary> ///采用https协议GET方式访问网络,根据传入的URl地址，得到响应的数据字符串。 ///</summary> ///<param name="URL">url地址</param> ///<param name="objencoding">编码方式例如：System.Text.Encoding.UTF8;</param> ///<returns>String类型的数据</returns> public string GetHtml(string URL, Encoding objencoding, string _Accept, string useragent) { //准备参数 SetRequest(URL, "GET", _Accept, "text/html", useragent, objencoding); //调用专门读取数据的类 return GetHttpRequestData(""); } ///<summary> ///采用https协议Post方式访问网络,根据传入的URl地址，得到响应的数据字符串。 ///</summary> ///<param name="URL">url地址</param> ///<param name="strPostdata">Post发送的数据</param> ///<param name="objencoding">编码方式例如：System.Text.Encoding.UTF8;</param> ///<returns>String类型的数据</returns> public string GetHtml(string URL, string strPostdata, Encoding objencoding) { //准备参数 SetRequest(URL, "post", "text/html, application/xhtml+xml, */*,zh-CN", "text/html", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", objencoding); //调用专门读取数据的类 return GetHttpRequestData(strPostdata); } #endregion } }
在很多情况下我们会使用间进程的webBrowser去实现一些网页的请求和抓去，这个时候有部分网页是取不到Cookie的，那怎么办呢？下面我提供一个方法，应该99%的都能取到，
//取当前webBrowser登录后的Cookie值 [DllImport("wininet.dll", CharSet = CharSet.Auto, SetLastError = true)] static extern bool InternetGetCookieEx(string pchURL, string pchCookieName, StringBuilder pchCookieData, ref int pcchCookieData, int dwFlags, object lpReserved); //取出Cookie，当登录后才能取 private static string GetCookieString(string url) { // Determine the size of the cookie int datasize = 256; StringBuilder cookieData = new StringBuilder(datasize); if (!InternetGetCookieEx(url, null, cookieData, ref datasize, 0x00002000, null)) { if (datasize < 0) return null; // Allocate stringbuilder large enough to hold the cookie cookieData = new StringBuilder(datasize); if (!InternetGetCookieEx(url, null, cookieData, ref datasize, 0x00002000, null)) return null; } return cookieData.ToString(); }
如果要使用中间的方法的话，可以访问我的帮助类完全免费开源：C# HttpHelper,帮助类,真正的Httprequest请求时无视编码，无视证书，无视Cookie,网页抓取
1.第一招，根据URL地址获取网页信息
先来看一下代码
get方法
public static string GetUrltoHtml(string Url,string type)
{
try
{
System.Net.WebRequest wReq = System.Net.WebRequest.Create(Url);
// Get the response instance.
System.Net.WebResponse wResp = wReq.GetResponse();
System.IO.Stream respStream = wResp.GetResponseStream();
// Dim reader As StreamReader = New StreamReader(respStream)
using (System.IO.StreamReader reader = new System.IO.StreamReader(respStream, Encoding.GetEncoding(type)))
{
return reader.ReadToEnd();
}
}
catch (System.Exception ex)
{
//errorMsg = ex.Message;
}
return "";
}
{
try
{
System.Net.WebRequest wReq = System.Net.WebRequest.Create(Url);
// Get the response instance.
System.Net.WebResponse wResp = wReq.GetResponse();
System.IO.Stream respStream = wResp.GetResponseStream();
// Dim reader As StreamReader = New StreamReader(respStream)
using (System.IO.StreamReader reader = new System.IO.StreamReader(respStream, Encoding.GetEncoding(type)))
{
return reader.ReadToEnd();
}
}
catch (System.Exception ex)
{
//errorMsg = ex.Message;
}
return "";
}
post方法
///<summary>
///采用https协议访问网络
///</summary>
///<param name="URL">url地址</param>
///<param name="strPostdata">发送的数据</param>
///<returns></returns>
public string OpenReadWithHttps(string URL, string strPostdata, string strEncoding)
{
Encoding encoding = Encoding.Default;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
request.Method = "post";
request.Accept = "text/html, application/xhtml+xml, */*";
request.ContentType = "application/x-www-form-urlencoded";
byte[] buffer = encoding.GetBytes(strPostdata);
request.ContentLength = buffer.Length;
request.GetRequestStream().Write(buffer, 0, buffer.Length);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
using( StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding(strEncoding)))
///采用https协议访问网络
///</summary>
///<param name="URL">url地址</param>
///<param name="strPostdata">发送的数据</param>
///<returns></returns>
public string OpenReadWithHttps(string URL, string strPostdata, string strEncoding)
{
Encoding encoding = Encoding.Default;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
request.Method = "post";
request.Accept = "text/html, application/xhtml+xml, */*";
request.ContentType = "application/x-www-form-urlencoded";
byte[] buffer = encoding.GetBytes(strPostdata);
request.ContentLength = buffer.Length;
request.GetRequestStream().Write(buffer, 0, buffer.Length);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
using( StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding(strEncoding)))
{
return reader.ReadToEnd();
return reader.ReadToEnd();
}
}
}
这招是入门第一式， 特点：
1.最简单最直观的一种，入门课程。
2.适应于明文，无需登录，无需任何验证就可以进入的页面。
3.获取的数据类型为HTML文档。
4.请求方法为Get/Post
2.第二招，根据URL地址获取需要验证证书才能访问的网页信息
先来看一下代码
get方法
//回调验证证书问题
public bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{ // 总是接受
return true;
}
/// <summary>
/// 传入URL返回网页的html代码
/// </summary>
/// <param name="Url">URL</param>
/// <returns></returns>
public string GetUrltoHtml(string Url)
{
StringBuilder content = new StringBuilder();
try
{
//这一句一定要写在创建连接的前面。使用回调的方法进行证书验证。
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);
// 与指定URL创建HTTP请求
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
//创建证书文件
X509Certificate objx509 = new X509Certificate(Application.StartupPath + "\\123.cer");
//添加到请求里
request.ClientCertificates.Add(objx509);
// 获取对应HTTP请求的响应
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
// 获取响应流
Stream responseStream = response.GetResponseStream();
// 对接响应流(以"GBK"字符集)
StreamReader sReader = new StreamReader(responseStream, Encoding.GetEncoding("utf-8"));
// 开始读取数据
Char[] sReaderBuffer = new Char[256];
int count = sReader.Read(sReaderBuffer, 0, 256);
while (count > 0)
{
String tempStr = new String(sReaderBuffer, 0, count);
content.Append(tempStr);
count = sReader.Read(sReaderBuffer, 0, 256);
}
// 读取结束
sReader.Close();
}
catch (Exception)
{
content = new StringBuilder("Runtime Error");
}
return content.ToString();
}
public bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{ // 总是接受
return true;
}
/// <summary>
/// 传入URL返回网页的html代码
/// </summary>
/// <param name="Url">URL</param>
/// <returns></returns>
public string GetUrltoHtml(string Url)
{
StringBuilder content = new StringBuilder();
try
{
//这一句一定要写在创建连接的前面。使用回调的方法进行证书验证。
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);
// 与指定URL创建HTTP请求
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
//创建证书文件
X509Certificate objx509 = new X509Certificate(Application.StartupPath + "\\123.cer");
//添加到请求里
request.ClientCertificates.Add(objx509);
// 获取对应HTTP请求的响应
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
// 获取响应流
Stream responseStream = response.GetResponseStream();
// 对接响应流(以"GBK"字符集)
StreamReader sReader = new StreamReader(responseStream, Encoding.GetEncoding("utf-8"));
// 开始读取数据
Char[] sReaderBuffer = new Char[256];
int count = sReader.Read(sReaderBuffer, 0, 256);
while (count > 0)
{
String tempStr = new String(sReaderBuffer, 0, count);
content.Append(tempStr);
count = sReader.Read(sReaderBuffer, 0, 256);
}
// 读取结束
sReader.Close();
}
catch (Exception)
{
content = new StringBuilder("Runtime Error");
}
return content.ToString();
}
post方法
//回调验证证书问题
public bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
// 总是接受
return true;
}
///<summary>
///采用https协议访问网络
///</summary>
///<param name="URL">url地址</param>
///<param name="strPostdata">发送的数据</param>
///<returns></returns>
public string OpenReadWithHttps(string URL, string strPostdata, string strEncoding)
{
// 这一句一定要写在创建连接的前面。使用回调的方法进行证书验证。
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);
Encoding encoding = Encoding.Default;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
//创建证书文件
X509Certificate objx509 = new X509Certificate(Application.StartupPath + "\\123.cer");
//加载Cookie
request.CookieContainer = new CookieContainer();
//添加到请求里
request.ClientCertificates.Add(objx509);
request.Method = "post";
request.Accept = "text/html, application/xhtml+xml, */*";
request.ContentType = "application/x-www-form-urlencoded";
byte[] buffer = encoding.GetBytes(strPostdata);
request.ContentLength = buffer.Length;
request.GetRequestStream().Write(buffer, 0, buffer.Length);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
using (StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding(strEncoding)))
public bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
// 总是接受
return true;
}
///<summary>
///采用https协议访问网络
///</summary>
///<param name="URL">url地址</param>
///<param name="strPostdata">发送的数据</param>
///<returns></returns>
public string OpenReadWithHttps(string URL, string strPostdata, string strEncoding)
{
// 这一句一定要写在创建连接的前面。使用回调的方法进行证书验证。
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);
Encoding encoding = Encoding.Default;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
//创建证书文件
X509Certificate objx509 = new X509Certificate(Application.StartupPath + "\\123.cer");
//加载Cookie
request.CookieContainer = new CookieContainer();
//添加到请求里
request.ClientCertificates.Add(objx509);
request.Method = "post";
request.Accept = "text/html, application/xhtml+xml, */*";
request.ContentType = "application/x-www-form-urlencoded";
byte[] buffer = encoding.GetBytes(strPostdata);
request.ContentLength = buffer.Length;
request.GetRequestStream().Write(buffer, 0, buffer.Length);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
using (StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding(strEncoding)))
{
return reader.ReadToEnd();
return reader.ReadToEnd();
}
}
}
这招是学会算是进了大门了，凡是需要验证证书才能进入的页面都可以使用这个方法进入，我使用的是证书回调验证的方式，证书验证是否通过在客户端验证，这样的话我们就可以使用自己定义一个方法来验证了，有的人会说那也不清楚是怎么样验证的啊，其它很简单，代码是自己写的为什么要那么难为自己呢，直接返回一个True不就完了，永远都是验证通过，这样就可以无视证书的存在了， 特点：
1.入门前的小难题，初级课程。
2.适应于无需登录，明文但需要验证证书才能访问的页面。
3.获取的数据类型为HTML文档。
4.请求方法为Get/Post
3.第三招，根据URL地址获取需要登录才能访问的网页信息
我们先来分析一下这种类型的网页，需要登录才能访问的网页，其它呢也是一种验证，验证什么呢，验证客户端是否登录，是否具用相应的凭证，需要登录的都要验证SessionID这是每一个需要登录的页面都需要验证的，那我们怎么做的，我们第一步就是要得存在Cookie里面的数据包括SessionID，那怎么得到呢，这个方法很多，使用ID9或者是火狐浏览器很容易就能得到，可以参考我的文章
提供一个网页抓取hao123手机号码归属地的例子 这里面针对ID9有详细的说明。
如果我们得到了登录的Cookie信息之后那个再去访问相应的页面就会非常的简单了，其它说白了就是把本地的Cookie信息在请求的时候捎带过去就行了。
看代码
get方法
View Code
/// <summary>
/// 传入URL返回网页的html代码带有证书的方法
/// </summary>
/// <param name="Url">URL</param>
/// <returns></returns>
public string GetUrltoHtml(string Url)
{
StringBuilder content = new StringBuilder();
try
{
// 与指定URL创建HTTP请求
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
request.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; BOIE9;ZHCN)";
request.Method = "GET";
request.Accept = "*/*";
//如果方法验证网页来源就加上这一句如果不验证那就可以不写了
request.Referer = "http://sufei.cnblogs.com";
CookieContainer objcok = new CookieContainer();
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("sidi_sessionid", "360A748941D055BEE8C960168C3D4233"));
request.CookieContainer = objcok;
//不保持连接
request.KeepAlive = true;
// 获取对应HTTP请求的响应
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
// 获取响应流
Stream responseStream = response.GetResponseStream();
// 对接响应流(以"GBK"字符集)
StreamReader sReader = new StreamReader(responseStream, Encoding.GetEncoding("gb2312"));
// 开始读取数据
Char[] sReaderBuffer = new Char[256];
int count = sReader.Read(sReaderBuffer, 0, 256);
while (count > 0)
{
String tempStr = new String(sReaderBuffer, 0, count);
content.Append(tempStr);
count = sReader.Read(sReaderBuffer, 0, 256);
}
// 读取结束
sReader.Close();
}
catch (Exception)
{
content = new StringBuilder("Runtime Error");
}
return content.ToString();
}
/// 传入URL返回网页的html代码带有证书的方法
/// </summary>
/// <param name="Url">URL</param>
/// <returns></returns>
public string GetUrltoHtml(string Url)
{
StringBuilder content = new StringBuilder();
try
{
// 与指定URL创建HTTP请求
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
request.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; BOIE9;ZHCN)";
request.Method = "GET";
request.Accept = "*/*";
//如果方法验证网页来源就加上这一句如果不验证那就可以不写了
request.Referer = "http://sufei.cnblogs.com";
CookieContainer objcok = new CookieContainer();
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("sidi_sessionid", "360A748941D055BEE8C960168C3D4233"));
request.CookieContainer = objcok;
//不保持连接
request.KeepAlive = true;
// 获取对应HTTP请求的响应
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
// 获取响应流
Stream responseStream = response.GetResponseStream();
// 对接响应流(以"GBK"字符集)
StreamReader sReader = new StreamReader(responseStream, Encoding.GetEncoding("gb2312"));
// 开始读取数据
Char[] sReaderBuffer = new Char[256];
int count = sReader.Read(sReaderBuffer, 0, 256);
while (count > 0)
{
String tempStr = new String(sReaderBuffer, 0, count);
content.Append(tempStr);
count = sReader.Read(sReaderBuffer, 0, 256);
}
// 读取结束
sReader.Close();
}
catch (Exception)
{
content = new StringBuilder("Runtime Error");
}
return content.ToString();
}
post方法。
View Code
///<summary>
///采用https协议访问网络
///</summary>
///<param name="URL">url地址</param>
///<param name="strPostdata">发送的数据</param>
///<returns></returns>
public string OpenReadWithHttps(string URL, string strPostdata)
{
Encoding encoding = Encoding.Default;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
request.Method = "post";
request.Accept = "text/html, application/xhtml+xml, */*";
request.ContentType = "application/x-www-form-urlencoded";
CookieContainer objcok = new CookieContainer();
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("sidi_sessionid", "360A748941D055BEE8C960168C3D4233"));
request.CookieContainer = objcok;
byte[] buffer = encoding.GetBytes(strPostdata);
request.ContentLength = buffer.Length;
request.GetRequestStream().Write(buffer, 0, buffer.Length);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding("utf-8"));
return reader.ReadToEnd();
}
///采用https协议访问网络
///</summary>
///<param name="URL">url地址</param>
///<param name="strPostdata">发送的数据</param>
///<returns></returns>
public string OpenReadWithHttps(string URL, string strPostdata)
{
Encoding encoding = Encoding.Default;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
request.Method = "post";
request.Accept = "text/html, application/xhtml+xml, */*";
request.ContentType = "application/x-www-form-urlencoded";
CookieContainer objcok = new CookieContainer();
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://sufei.cnblogs.com"), new Cookie("sidi_sessionid", "360A748941D055BEE8C960168C3D4233"));
request.CookieContainer = objcok;
byte[] buffer = encoding.GetBytes(strPostdata);
request.ContentLength = buffer.Length;
request.GetRequestStream().Write(buffer, 0, buffer.Length);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.GetEncoding("utf-8"));
return reader.ReadToEnd();
}
特点：
1.还算有点水类型的，练习成功后可以小牛一把。
2.适应于需要登录才能访问的页面。
3.获取的数据类型为HTML文档。
4.请求方法为Get/Post
总结一下，其它基本的技能就这几个部分，如果再深入的话那就是基本技能的组合了
比如，
1. 先用Get或者Post方法登录然后取得Cookie再去访问页面得到信息，这种其它也是上面技能的组合，
这里需要以请求后做这样一步
response.Cookies
这就是在你请求后可以得到当次Cookie的方法，直接取得返回给上一个方法使用就行了，上面我们都是自己构造的，在这里直接使用这个Cookie就可以了。
2.如果我们碰到需要登录而且还要验证证书的网页怎么办，其它这个也很简单把我们上面的方法综合 一下就行了
如下代码这里我以Get为例子Post例子也是同样的方法
View Code
/// <summary>
/// 传入URL返回网页的html代码
/// </summary>
/// <param name="Url">URL</param>
/// <returns></returns>
public string GetUrltoHtml(string Url)
{
StringBuilder content = new StringBuilder();
try
{
//这一句一定要写在创建连接的前面。使用回调的方法进行证书验证。
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);
// 与指定URL创建HTTP请求
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
//创建证书文件
X509Certificate objx509 = new X509Certificate(Application.StartupPath + "\\123.cer");
//添加到请求里
request.ClientCertificates.Add(objx509);
CookieContainer objcok = new CookieContainer();
objcok.Add(new Uri("http://www.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://www.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://www.cnblogs.com"), new Cookie("sidi_sessionid", "360A748941D055BEE8C960168C3D4233"));
request.CookieContainer = objcok;
// 获取对应HTTP请求的响应
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
// 获取响应流
Stream responseStream = response.GetResponseStream();
// 对接响应流(以"GBK"字符集)
StreamReader sReader = new StreamReader(responseStream, Encoding.GetEncoding("utf-8"));
// 开始读取数据
Char[] sReaderBuffer = new Char[256];
int count = sReader.Read(sReaderBuffer, 0, 256);
while (count > 0)
{
String tempStr = new String(sReaderBuffer, 0, count);
content.Append(tempStr);
count = sReader.Read(sReaderBuffer, 0, 256);
}
// 读取结束
sReader.Close();
}
catch (Exception)
{
content = new StringBuilder("Runtime Error");
}
return content.ToString();
}
/// 传入URL返回网页的html代码
/// </summary>
/// <param name="Url">URL</param>
/// <returns></returns>
public string GetUrltoHtml(string Url)
{
StringBuilder content = new StringBuilder();
try
{
//这一句一定要写在创建连接的前面。使用回调的方法进行证书验证。
ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);
// 与指定URL创建HTTP请求
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
//创建证书文件
X509Certificate objx509 = new X509Certificate(Application.StartupPath + "\\123.cer");
//添加到请求里
request.ClientCertificates.Add(objx509);
CookieContainer objcok = new CookieContainer();
objcok.Add(new Uri("http://www.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://www.cnblogs.com"), new Cookie("键", "值"));
objcok.Add(new Uri("http://www.cnblogs.com"), new Cookie("sidi_sessionid", "360A748941D055BEE8C960168C3D4233"));
request.CookieContainer = objcok;
// 获取对应HTTP请求的响应
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
// 获取响应流
Stream responseStream = response.GetResponseStream();
// 对接响应流(以"GBK"字符集)
StreamReader sReader = new StreamReader(responseStream, Encoding.GetEncoding("utf-8"));
// 开始读取数据
Char[] sReaderBuffer = new Char[256];
int count = sReader.Read(sReaderBuffer, 0, 256);
while (count > 0)
{
String tempStr = new String(sReaderBuffer, 0, count);
content.Append(tempStr);
count = sReader.Read(sReaderBuffer, 0, 256);
}
// 读取结束
sReader.Close();
}
catch (Exception)
{
content = new StringBuilder("Runtime Error");
}
return content.ToString();
}
3.如果我们碰到那种需要验证网页来源的方法应该怎么办呢，这种情况其它是有些程序员会想到你可能会使用程序，自动来获取网页信息，为了防止就使用页面来源来验证，就是说只要不是从他们所在页面或是域名过来的请求就不接受，有的是直接验证来源的IP，这些都可以使用下面一句来进入，这主要是这个地址是可以直接伪造的
request.Referer = "http://sufei.cnblogs.com";
呵呵其它很简单因为这个地址可以直接修改。但是如果服务器上验证的是来源的URL那就完了，我们就得去修改数据包了，这个有点难度暂时不讨论。
4.提供一些与这个例子相配置的方法
过滤HTML标签的方法
View Code
/// <summary>
/// 过滤html标签
/// </summary>
/// <param name="strHtml">html的内容</param>
/// <returns></returns>
public static string StripHTML(string stringToStrip)
{
// paring using RegEx //
stringToStrip = Regex.Replace(stringToStrip, "</p(?:\\s*)>(?:\\s*)<p(?:\\s*)>", "\n\n", RegexOptions.IgnoreCase | RegexOptions.Compiled);
stringToStrip = Regex.Replace(stringToStrip, "<br(?:\\s*)/>", "\n", RegexOptions.IgnoreCase | RegexOptions.Compiled);
stringToStrip = Regex.Replace(stringToStrip, "\"", "''", RegexOptions.IgnoreCase | RegexOptions.Compiled);
stringToStrip = StripHtmlXmlTags(stringToStrip);
return stringToStrip;
}
private static string StripHtmlXmlTags(string content)
{
return Regex.Replace(content, "<[^>]+>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
}
/// 过滤html标签
/// </summary>
/// <param name="strHtml">html的内容</param>
/// <returns></returns>
public static string StripHTML(string stringToStrip)
{
// paring using RegEx //
stringToStrip = Regex.Replace(stringToStrip, "</p(?:\\s*)>(?:\\s*)<p(?:\\s*)>", "\n\n", RegexOptions.IgnoreCase | RegexOptions.Compiled);
stringToStrip = Regex.Replace(stringToStrip, "<br(?:\\s*)/>", "\n", RegexOptions.IgnoreCase | RegexOptions.Compiled);
stringToStrip = Regex.Replace(stringToStrip, "\"", "''", RegexOptions.IgnoreCase | RegexOptions.Compiled);
stringToStrip = StripHtmlXmlTags(stringToStrip);
return stringToStrip;
}
private static string StripHtmlXmlTags(string content)
{
return Regex.Replace(content, "<[^>]+>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
}
URL转化的方法
#region 转化 URL
public static string URLDecode(string text)
{
return HttpUtility.UrlDecode(text, Encoding.Default);
}
public static string URLEncode(string text)
{
return HttpUtility.UrlEncode(text, Encoding.Default);
}
#endregion
public static string URLDecode(string text)
{
return HttpUtility.UrlDecode(text, Encoding.Default);
}
public static string URLEncode(string text)
{
return HttpUtility.UrlEncode(text, Encoding.Default);
}
#endregion
提供一个实际例子，这个是使用IP138来查询手机号码归属地的方法，其它在我的上一次文章里都有，在这里我再放上来是方便大家阅读，这方面的技术其它研究起来很有意思，希望大家多提建议，我相信应该还有更多更好，更完善的方法，在这里给大家提供一个参考吧。感谢支持
上例子
/// <summary>
/// 输入手机号码得到归属地信息
/// </summary>
/// <param name="number">手机号码</param>
/// <returns>数组类型0为归属地，1卡类型，2区 号，3邮 编</returns>
public static string[] getTelldate(string number)
{
try
{
string strSource = GetUrltoHtml("http://www.ip138.com:8080/search.asp?action=mobile&mobile=" + number.Trim());
//归属地
strSource = strSource.Substring(strSource.IndexOf(number));
strSource = StripHTML(strSource);
strSource = strSource.Replace("\r", "");
strSource = strSource.Replace("\n", "");
strSource = strSource.Replace("\t", "");
strSource = strSource.Replace(" ", "");
strSource = strSource.Replace("-->", "");
string[] strnumber = strSource.Split(new string[] { "归属地", "卡类型", "邮 编", "区 号", "更详细", "卡号" }, StringSplitOptions.RemoveEmptyEntries);
string[] strnumber1 = null;
if (strnumber.Length > 4)
{
strnumber1 = new string[] { strnumber[1].Trim(), strnumber[2].Trim(), strnumber[3].Trim(), strnumber[4].Trim() };
}
return strnumber1;
}
catch (Exception)
{
return null;
}
}
/// 输入手机号码得到归属地信息
/// </summary>
/// <param name="number">手机号码</param>
/// <returns>数组类型0为归属地，1卡类型，2区 号，3邮 编</returns>
public static string[] getTelldate(string number)
{
try
{
string strSource = GetUrltoHtml("http://www.ip138.com:8080/search.asp?action=mobile&mobile=" + number.Trim());
//归属地
strSource = strSource.Substring(strSource.IndexOf(number));
strSource = StripHTML(strSource);
strSource = strSource.Replace("\r", "");
strSource = strSource.Replace("\n", "");
strSource = strSource.Replace("\t", "");
strSource = strSource.Replace(" ", "");
strSource = strSource.Replace("-->", "");
string[] strnumber = strSource.Split(new string[] { "归属地", "卡类型", "邮 编", "区 号", "更详细", "卡号" }, StringSplitOptions.RemoveEmptyEntries);
string[] strnumber1 = null;
if (strnumber.Length > 4)
{
strnumber1 = new string[] { strnumber[1].Trim(), strnumber[2].Trim(), strnumber[3].Trim(), strnumber[4].Trim() };
}
return strnumber1;
}
catch (Exception)
{
return null;
}
}
这个例子写是不怎么样，些地方是可以简化的，这个接口而且可以直接使用Xml得到，但我在这里的重点是让一些新手看看方法和思路风凉啊，呵呵
第四招，通过Socket访问
------------------------------------------------------------------------------------------------------------
///<summary>
/// 请求的公共类用来向服务器发送请求
///</summary>
///<param name="strSMSRequest">发送请求的字符串</param>
///<returns>返回的是请求的信息</returns>
private static string SMSrequest(string strSMSRequest)
{
byte[] data = new byte[1024];
string stringData = null;
IPHostEntry gist = Dns.GetHostByName("www.110.cn");
IPAddress ip = gist.AddressList[0];
//得到IP
IPEndPoint ipEnd = new IPEndPoint(ip, 3121);
//默认80端口号
Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
//使用tcp协议 stream类型
try
{
socket.Connect(ipEnd);
}
catch (SocketException ex)
{
return "Fail to connect server\r\n" + ex.ToString();
}
string path = strSMSRequest.ToString().Trim();
StringBuilder buf = new StringBuilder();
//buf.Append("GET ").Append(path).Append(" HTTP/1.0\r\n");
//buf.Append("Content-Type: application/x-www-form-urlencoded\r\n");
//buf.Append("\r\n");
byte[] ms = System.Text.UTF8Encoding.UTF8.GetBytes(buf.ToString());
//提交请求的信息
socket.Send(ms);
//接收返回
string strSms = "";
int recv = 0;
do
{
recv = socket.Receive(data);
stringData = Encoding.ASCII.GetString(data, 0, recv);
//如果请求的页面meta中指定了页面的encoding为gb2312则需要使用对应的Encoding来对字节进行转换()
strSms = strSms + stringData;
//strSms += recv.ToString();
}
while (recv != 0);
socket.Shutdown(SocketShutdown.Both);
socket.Close();
return strSms;
}
浙公网安备 33010602011771号