【爬虫入门】HttpClient+Jsoup进行简单的网页访问和信息保存
【项目选型】
（Maven）SpringBoot+JPA
【项目搭建】
pom.xml:
<parent>
<artifactId>spring-boot-starter-parent</artifactId>
<groupId>org.springframework.boot</groupId>
<version>2.5.0</version>
</parent>
<dependencies>
<!--spring-boot-mvc-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!--springData JPA-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<!--mysql-->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.14.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
</dependencies>
AppMain.class:
@SpringBootApplication /** * 开启定时任务 */ @EnableScheduling public class AppMain { public static void main(String[] args) { SpringApplication.run(AppMain.class,args); } }
【分析】
【具体实现】
POJO+JPA+SQL建表
POJO类： @Table(name = "md_item") @Entity public class Product { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) String id; String proid; String proauthor; String protitle; String probackcount; String probackermoney; String promoneypercent; String starttime; String endtime; String protype; String prostatus; String proimgpath; // get/set/toString } JPA： /** * extends JpaRepository<PoJo类,Key主键类型> */ public interface ProductDao extends JpaRepository<Product,Long> { } SQL： DROP TABLE IF EXISTS `md_item`; CREATE TABLE `md_item` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `proid` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `proauthor` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `protitle` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `probackcount` varchar(20) DEFAULT NULL, `probackermoney` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `promoneypercent` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `starttime` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `endtime` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `protype` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `prostatus` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `proimgpath` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=82 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
封装HttpClient 实现获取网页代码和图片下载
/** * 封装HttpClient，交给Spring容器管理 */ @Component public class HttpUtils { //httpclient连接池！ private PoolingHttpClientConnectionManager clientConnectionManager; /** * 在构造方法中new一个 */ public HttpUtils() { this.clientConnectionManager = new PoolingHttpClientConnectionManager(); //设置最大连接数 this.clientConnectionManager.setMaxTotal(100); //设置每个主机的最大连接数 this.clientConnectionManager.setDefaultMaxPerRoute(10); } /** * 使用get请求获得页面 * @param url * @return 页面数据 */ public String doGetHtml(String url){ //获取HttpClient对象 CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.clientConnectionManager).build(); //创建HttpGet请求对象，设置url地址 HttpGet httpGet = new HttpGet(url); //设置请求信息 httpGet.setConfig(this.getConfig()); CloseableHttpResponse httpResponse=null; //使用HttpClient发起请求，获得相应 try { httpResponse = httpClient.execute(httpGet); if(httpResponse.getStatusLine().getStatusCode()==200){ //判断Entity是否为空，如果不为空就可以使用EntityUtils if(httpResponse.getEntity()!=null){ String content = EntityUtils.toString(httpResponse.getEntity(),"utf-8"); return content; }else{ return "ERROR"; } } } catch (IOException e) { e.printStackTrace(); }finally { if(httpResponse!=null){ try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } } //解析响应，返回结果 return ""; } /** * 设置RequestConfig * @return */ private RequestConfig getConfig() { RequestConfig config=RequestConfig.custom() .setConnectTimeout(1000) //创建链接的最长时间 .setConnectionRequestTimeout(500) //获取链接的最长时间 .setSocketTimeout(10000) //数据传输的最长时间 .build(); return config; } /** * 下载图片 * @param url * @return 图片名称 */ public String doGetImage(String url){ //获取HttpClient对象 CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.clientConnectionManager).build(); //创建HttpGet请求对象，设置url地址 HttpGet httpGet = new HttpGet(url); //设置请求信息 httpGet.setConfig(this.getConfig()); CloseableHttpResponse httpResponse=null; //使用HttpClient发起请求，获得相应 try { httpResponse = httpClient.execute(httpGet); if(httpResponse.getStatusLine().getStatusCode()==200){ //判断Entity是否为空，如果不为空就可以使用EntityUtils if(httpResponse.getEntity()!=null){ //下载图片 //获取图片后缀 String exName=url.substring(url.lastIndexOf(".")); //创建图片名，重命名图片 String picName= UUID.randomUUID().toString()+exName; //下载图片 OutputStream outputStream=new FileOutputStream(new File("G:/IJDailyCode/Crawler/src/main/resources/downloadImg/"+picName)); httpResponse.getEntity().writeTo(outputStream); //返回图片名称 return picName; }else{ return "ERROR"; } } } catch (IOException e) { e.printStackTrace(); }finally { if(httpResponse!=null){ try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } } //出现其他问题返回结果 return ""; }
任务方法：
@Component public class GetTask { @Autowired HttpUtils httpUtils; @Autowired ProductService productService; @Scheduled(fixedDelay = 100*1000) public void crawlerMain(){ String mainUrl="https://zhongchou.modian.com/all/top_time/going/"; /*页码for循环*/ for(int page=1;page<=8;++page){ String tempUrl=mainUrl+page; String html = httpUtils.doGetHtml(tempUrl); /*解析页面，获取商品数据*/ this.parse(html); } } private void parse(String html) { Document doc = Jsoup.parse(html); Elements proElms = doc.select("div.pro_field > ul > li"); for (Element proElm:proElms) { /** * 乱七八糟的搜寻匹配项 */ String proId = proElm.attr("data-pro-id"); String proAuthor=proElm.select("div.author > a > p").text(); // 重复的使用选择器找信息。。。。。。。。。。。。。 String imgUrl=infoDoc.getElementById("big_logo").attr("src"); //如果实体的属性是null,它就会忽略它,这里只传一个proId参数就好 Product proExample=new Product(); proExample.setProid(proId); //查询并判断数据是否存在 List<Product> examples = productService.findAll(proExample); System.out.println("list有无数据:"+examples.size()); if(examples.size()>0){ System.out.println("===数据已存在==="); continue; } /*绑定数值*/ proExample.setProauthor(proAuthor); // 重复的数值绑定操作。。。。。。。。。 proExample.setProstatus(proStatus); /*下载图片*/ String proImgPath=httpUtils.doGetImage(imgUrl); proExample.setProimgpath(proImgPath); /*提交保存*/ productService.save(proExample); } } }
最没用的Service
@Service @Transactional public class ProductServiceImpl implements ProductService { @Autowired private ProductDao productDao; @Override public void save(Product product) { this.productDao.save(product); } @Override public List<Product> findAll(Product product) { Example<Product> example = Example.of(product); List<Product> products = productDao.findAll(example); return products; } }
【保存结果展示】
【遇到问题】
网页get下来本身的数据就是NULL，无法判断，导致保存失败。懒得加判断了。
【重要】在判断数据是否重复时，使用JPARepository的findAll的Example方式，提供一个POJO模板进行自动查询，返回的List.size()始终为0，导致重复数据还会保存。
【重要】@Table(name=" 表名")报红
