爬取链家、贝壳、大唐二手房数据

一、大唐二手房

public List<HouseInfo> GetData(int page = 1)
        {
            string baseurl = $"https://datang.anjuke.com/gongsi-esf/p{page}";
            HtmlWeb web = new HtmlWeb();
            HtmlDocument doc = web.Load(baseurl);
            HtmlNode ul = doc.GetElementbyId("houselist-mod");
            var list = new List<HouseInfo>();
            foreach (var li in ul.ChildNodes)
            {
                try
                {
                    if (!li.Name.ToLower().Contains("li")) continue;
                    var itemInfo = li.SelectSingleNode("div[@class='house-details']");
                    var url = itemInfo.SelectSingleNode("div[@class='house-title']").ChildNodes[1].Attributes["href"].Value;
                    var title = itemInfo.SelectSingleNode("div[@class='house-title']").InnerText.Replace("\r", "").Replace("\n", "");
                    var detail = itemInfo.SelectNodes("div[@class='details-item']");
                    var info = detail[0].InnerText.Replace("\r", "").Replace("\n", "");
                    var addressInfo = detail[1].InnerText.Replace("\r", "").Replace("\n", "");
                    var tag = itemInfo.SelectSingleNode("div[@class='tags-bottom']").InnerText.Replace("\r", "").Replace("\n", "");

                    var priceInfo = li.SelectSingleNode("div[@class='pro-price']");
                    var amount = priceInfo.SelectSingleNode("span[@class='price-det']").InnerText.Replace("\r", "").Replace("\n", "");
                    var price = priceInfo.SelectSingleNode("span[@class='unit-price']").InnerText.Replace("\r", "").Replace("\n", "");
                    list.Add(new HouseInfo
                    {
                        title = title.Trim(),
                        plot = addressInfo.Split("&nbsp;")[0].Trim(),
                        tag = tag,
                        url = url,
                        address = addressInfo.Split("&nbsp;")[1].Trim(),
                        addressInfo = addressInfo.Trim(),
                        area = info.Split('|')[1].Trim(),
                        housetype = info.Split('|')[0].Trim(),
                        floortype = info.Split("|")[2].Trim(),
                        year = info.Split("|")[3].Split(" &#xE147;")[0].Trim(),
                        itemInfo = info.Trim(),
                        city = addressInfo.Split("&nbsp;")[1].Split("-")[0],
                        amount = amount.Trim(),
                        price = price.Trim()
                    });
                }
                catch (Exception)
                {

                }   
            }
            return list;
        }

二、贝壳二手房

public List<HouseInfo> GetData(int page = 1)
        {
            string baseurl = $"https://cd.ke.com/ershoufang/pg{page}";
            HtmlWeb web = new HtmlWeb();
            HtmlDocument doc = web.Load(baseurl);
            HtmlNode ul = doc.DocumentNode.SelectSingleNode("//ul[@class='sellListContent']");
            var list = new List<HouseInfo>();
            foreach (var li in ul.ChildNodes)
            {
                try
                {
                    if (!li.HasClass("clear")) continue;
                    var itemInfo = li.SelectSingleNode("div[@class='info clear']");
                    var url = itemInfo.SelectSingleNode("div[@class='title']").ChildNodes[1].Attributes["href"].Value;
                    var title = itemInfo.SelectSingleNode("div[@class='title']").InnerText.Replace("\r", "").Replace("\n", "");

                    var detail = itemInfo.SelectSingleNode("div[@class='address']");
                    var plot = detail.SelectSingleNode("div[@class='flood']").InnerText.Replace("\r", "").Replace("\n", "");

                    var info = detail.SelectSingleNode("div[@class='houseInfo']").InnerText.Replace("\r", "").Replace("\n", "");
                    var tag = detail.SelectSingleNode("div[@class='tag']").InnerText.Replace("\r", "").Replace("\n", "");

                    var priceInfo = detail.SelectSingleNode("div[@class='priceInfo']");
                    var amount = priceInfo.SelectSingleNode("div[@class='totalPrice']").InnerText.Replace("\r", "").Replace("\n", "");
                    var price = priceInfo.SelectSingleNode("div[@class='unitPrice']").InnerText.Replace("\r", "").Replace("\n", "");
                    list.Add(new HouseInfo
                    {
                        title = title.Trim(),
                        plot = plot.Trim(),
                        tag = tag.Trim(),
                        url = url.Trim(),
                        area = info.Split('|')[3].Trim(),
                        toward = info.Split('|')[3].Trim(),
                        housetype = info.Split('|')[2].Trim(),
                        floortype = info.Split("|")[0].Trim(),
                        year = info.Split("|")[1].Trim(),
                        itemInfo = info.Trim(),
                        amount = amount.Trim(),
                        price = price.Trim()
                    });
                }
                catch (Exception)
                {

                }
            }
            return list;
        }

三、链家二手房

public List<HouseInfo> GetData(int page = 1)
        {
            string baseurl = $"https://cd.lianjia.com/ershoufang/pg{page}";
            HtmlWeb web = new HtmlWeb();
            HtmlDocument doc = web.Load(baseurl);
            HtmlNode ul = doc.DocumentNode.SelectSingleNode("//ul[@class='sellListContent']");
            var list = new List<HouseInfo>();
            foreach (var li in ul.ChildNodes)
            {
                try
                {
                    if (!li.HasClass("clear")) continue;
                    var itemInfo = li.SelectSingleNode("div[@class='info clear']");
                    var url = itemInfo.SelectSingleNode("div[@class='title']").ChildNodes[0].Attributes["href"].Value;
                    var title = itemInfo.SelectSingleNode("div[@class='title']").InnerText?.Replace("\r", "").Replace("\n", "");

                    var detail = itemInfo.SelectSingleNode("div[@class='address']");
                    var plot = itemInfo.SelectSingleNode("div[@class='flood']").InnerText?.Replace("\r", "").Replace("\n", "");

                    var info = detail.SelectSingleNode("div[@class='houseInfo']").InnerText?.Replace("\r", "").Replace("\n", "");
                    var tag = itemInfo.SelectSingleNode("div[@class='tag']").InnerText?.Replace("\r", "").Replace("\n", "");

                    var priceInfo = itemInfo.SelectSingleNode("div[@class='priceInfo']");
                    var amount = priceInfo.SelectSingleNode("div[@class='totalPrice']").InnerText?.Replace("\r", "").Replace("\n", "");
                    var price = priceInfo.SelectSingleNode("div[@class='unitPrice']").InnerText?.Replace("\r", "").Replace("\n", "");
                    list.Add(new HouseInfo
                    {
                        title = title.Trim(),
                        plot = plot.Trim(),
                        city=plot.Split('-').Length>1?plot.Split('-')[1].Trim():"",
                        tag = tag.Trim(),
                        url = url.Trim(),
                        area = info.Split('|')[1].Trim(),
                        toward = info.Split('|')[3].Trim(),
                        housetype = info.Split('|')[0].Trim(),
                        floortype = info.Split("|")[4].Trim(),
                        year = info.Split("|")[5].Trim(),
                        itemInfo = info.Trim(),
                        amount = amount.Trim(),
                        price = price.Trim()
                    });
                }
                catch (Exception)
                {

                }
            }
            return list;
        }

 

四、实体类

public class HouseInfo
    {
        /// <summary>
        /// 标题
        /// </summary>
        public string title { get; set; }
        /// <summary>
        /// 详情
        /// </summary>
        public string itemInfo { get; set; }
        /// <summary>
        /// 区域
        /// </summary>
        public string addressInfo { get; set; }
        /// <summary>
        /// 地址
        /// </summary>
        public string url { get; set; }
        /// <summary>
        /// 房屋类型
        /// </summary>
        public string housetype { get; set; }
        /// <summary>
        /// 面积
        /// </summary>
        public string area { get; set; }
        /// <summary>
        /// 楼层类型
        /// </summary>
        public string floortype { get; set; }
        /// <summary>
        /// 建设年份
        /// </summary>
        public string year { get; set; }
        /// <summary>
        /// 区域
        /// </summary>
        public string city { get; set; }
        /// <summary>
        /// 具体地址
        /// </summary>
        public string address { get; set; }
        /// <summary>
        /// 小区
        /// </summary>
        public string plot { get; set; }
        /// <summary>
        /// 标签
        /// </summary>
        public string tag { get; set; }
        /// <summary>
        /// 总价
        /// </summary>
        public string amount { get; set; }
        /// <summary>
        /// 面积
        /// </summary>
        public string price { get; set; }
        /// <summary>
        /// 朝向
        /// </summary>
        public string toward { get; set; }
    }