C#访问网页，得到数据

读取网页内容，一个C#的爬虫的基本步骤

里面需要的request.UserAgent需要设置一下，有些网站需要这个参数

当时写的时候没有写注释。也是后来用的时候很方便。

只要传递一个网址过去，就可以返回这个网址的HTML源代码

  /// <summary>
        /// 得到网页数据
        /// </summary>
        /// <returns>网页</returns>
         protected string GetPageContent(string getpage)
        {
            try
            {
                CookieContainer cc = new CookieContainer();
                string content = string.Empty;
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(getpage);
                request.Method = "post";
                request.Timeout = 10000;
                request.ContentType = "application/x-www-form-urlencoded";
                request.ContentLength = content.Length;//????
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322)";
                //request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13";
                request.CookieContainer = cc;

                Stream stream = request.GetRequestStream();

                StreamWriter writer = new StreamWriter(stream, Encoding.GetEncoding("gb2312"));
                writer.Write(content, 0, content.Length);
                writer.Close();
                stream.Close();

                HttpWebResponse response = request.GetResponse() as HttpWebResponse;
                response.Cookies = cc.GetCookies(request.RequestUri);
                Stream readerStream = response.GetResponseStream();
                StreamReader sr = new StreamReader(readerStream, Encoding.GetEncoding("gb2312"));
                content = sr.ReadToEnd();
                sr.Close();
                readerStream.Close();
                return content;
            }
            catch (Exception)
            {
                return null;
            }

        }