C#实现页面抓取

方法一：WebClient
WebClient wc = new WebClient();
wc.Credentials = CredentialCache.DefaultCredentials;
Byte[] pageData = wc.DownloadData(PageUrl);

方法二：WebRequest
WebRequest request = WebRequest.Create(PageUrl);
WebResponse response = request.GetResponse();
Stream resStream = response.GetResponseStream();

方法三：使用第三方工具

以下仅用方法2实现：

private void Page_Load(object sender, System.EventArgs e)
        {
            // 在此处放置用户代码以初始化页面

            
            HttpWebRequest req = (HttpWebRequest) WebRequest.Create("http://www.bank-of-china.com/cn/common/service.jsp?category=ROOT%3E%D6%D0%D0%D0%D7%DC%D0%D0%3E%D0%D0%C7%E9%B7%D6%CE%F6%CA%D2%3E%D6%D0%D0%D0%BB%E3%C6%C0%3E%CD%E2%BB%E3%C5%C6%BC%DB");
            req.AllowAutoRedirect = true;
            req.MaximumAutomaticRedirections = 3;
            req.UserAgent = "Mozilla/6.0 (MSIE 6.0; Windows NT 5.1; Natas.Robot)";
            req.KeepAlive = true;
            req.Timeout = 4000;
    
            // Get the stream from the returned web response
            HttpWebResponse webresponse = null;
            try 
            {
                webresponse = (HttpWebResponse) req.GetResponse();
            } 
            catch(System.Net.WebException ex) 
            {
                string message = "error response exception:" + ex.Message;
                Console.WriteLine(message);
            }
            if (webresponse != null) 
            {
                StreamReader stream = new StreamReader
                    (webresponse.GetResponseStream(), Encoding.GetEncoding("GB2312") );
                
                string strAll;
                
                strAll="";
                int len;

                strAll=stream.ReadToEnd();
                len=strAll.IndexOf("货币名称",0);
                strAll=strAll.Substring(len,strAll.Length-len);
                len=strAll.IndexOf("免责声明",0);
                strAll=strAll.Substring(0,len);
            
                
                //Bussiness_Process
                string strBuss;
                string strA;
                strBuss="";
                strA="";
                for(int i=0;i<strAll.Length;i++)
                {
                    strA=strAll.Substring(i,1);
                    if(strA=="<")
                    {
                        i+=1;
                        strA=strAll.Substring(i,1);
                        while(strA!=">")
                        {
                            i+=1;
                            strA=strAll.Substring(i,1);
                        }
                    }
                    else
                    {
                        strBuss=strBuss + strA + " ";
                    }
                }

                Tv_content.Text=strBuss;
                stream.Close();
                

            }

        }

还可加入的属性 
    req.ProtocolVersion=HttpVersion.Version10;

    req.Method = "GET";

    req.ContentType = "application/x-www-form-urlencoded";

    req.Proxy = System.Net.WebProxy.GetDefaultProxy();