数据采集知识

最近在采集数据所以自己写了个小工具，一般的网站可以直接模拟浏览器发送请求，然后用正则表达式分析页面提取需要的信息。

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
request.Timeout = 5000;
request.UserAgent="Mozilla/5.0 (Windows NT 5.2; rv:27.0) Gecko/20100101 Firefox/27.0";
request.Accept = "    text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
request.Headers.Add("Accept-Encoding", "gzip, deflate");
request.KeepAlive = true;
request.Referer = System.Web.HttpUtility.UrlEncode(url);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
string html = string.Empty;
if (response.ContentEncoding.ToLower() == "gzip")//如果使用了GZip则先解压
{
    using (System.IO.Stream streamReceive = response.GetResponseStream())
    {
        using (var zipStream =
            new System.IO.Compression.GZipStream(streamReceive, System.IO.Compression.CompressionMode.Decompress))
        {
            using (StreamReader sr = new System.IO.StreamReader(zipStream, Encoding.GetEncoding("gbk")))
            {
                html = sr.ReadToEnd();
            }
        }
    }
}
else
{
    using (System.IO.Stream streamReceive = response.GetResponseStream())
    {
        using (System.IO.StreamReader sr = new System.IO.StreamReader(streamReceive, Encoding.GetEncoding("gbk")))
        {
            html = sr.ReadToEnd();
        }
    }
}

string pattern = "<div>(\\d+)</div>";
Regex reg = new Regex(pattern, RegexOptions.IgnoreCase);
MatchCollection matchs = reg.Matches(html);

foreach (Match match in matchs)
{
    string phone = match.Groups[1].Value;
}