JAVA网络爬虫
HttpClient

导航

 
HttpClient爬取网页

HttpClient

项目结构

在这里插入图片描述

配置文件

  1. pom.xml

    <?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>cn.xiaoge</groupId>
    <artifactId>xiaoge-crawler-first</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>

        <!-- 自动的抓取数据的jar包, httpclient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>

        <!-- 日志信息 -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.25</version>
            <!--<scope>test</scope>-->
        </dependency>

    </dependencies>

</project>

  2. log4j.properties

    log4j.rootLogger=DEBUG,A1
log4j.logger.cn.itcast=DEBUG

# 在控制台显示日志
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

操作类

  1. HttpGetTest(不带参Get)

    package com.xiaoge.crawler.test;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @Author: 潇哥
 * @DateTime: 2020/9/21 下午4:35
 * @Description: TODO
 */
public class HttpGetTest {

    public static void main(String[] args) {

        // 1. 创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 2. 创建HttpGet对象, 设置url访问地址
        HttpGet httpGet = new HttpGet("https://www.csdn.net/");

        CloseableHttpResponse httpResponse = null;

        try{
            // 3. 使用HttpClient发起请求, 获取response
             httpResponse = httpClient.execute(httpGet);

            // 4. 解析响应
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                // 获取响应体
                HttpEntity httpEntity = httpResponse.getEntity();

                String content = EntityUtils.toString(httpEntity, "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            // 关闭httpResponse
            if (httpResponse != null) {
                try {
                    httpResponse.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            // 关闭httpClient
            if (httpClient != null) {
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }




    }

}

  2. HttpGetParamTest(带参Get)

    package com.xiaoge.crawler.test;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @Author: 潇哥
 * @DateTime: 2020/9/21 下午4:35
 * @Description: TODO
 */
public class HttpGetParamTest {

    public static void main(String[] args) throws Exception {

        // 1. 创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 设置请求地址是: https://so.csdn.net/so/search/s.do?q=java
        // 创建URIBuilder
        URIBuilder uriBuilder = new URIBuilder("https://so.csdn.net/so/search/s.do");

        // 设置参数, 如果是多个参数继续setParameter
        uriBuilder.setParameter("q", "java");

        // 2. 创建HttpGet对象, 设置url访问地址
        HttpGet httpGet = new HttpGet(uriBuilder.build());

        CloseableHttpResponse httpResponse = null;

        try{
            // 3. 使用HttpClient发起请求, 获取response
             httpResponse = httpClient.execute(httpGet);

            // 4. 解析响应
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                // 获取响应体
                HttpEntity httpEntity = httpResponse.getEntity();

                String content = EntityUtils.toString(httpEntity, "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            // 关闭httpResponse
            if (httpResponse != null) {
                try {
                    httpResponse.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            // 关闭httpClient
            if (httpClient != null) {
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }




    }

}

  3. HttpPostTest(不带参post)

    package com.xiaoge.crawler.test;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @Author: 潇哥
 * @DateTime: 2020/9/21 下午4:35
 * @Description: TODO
 */
public class HttpPostTest {

    public static void main(String[] args) {

        // 1. 创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 2. 创建HttpPost对象, 设置url访问地址
        HttpPost httpPost = new HttpPost("https://www.csdn.net/");

        CloseableHttpResponse httpResponse = null;

        try{
            // 3. 使用HttpClient发起请求, 获取response
             httpResponse = httpClient.execute(httpPost);

            // 4. 解析响应
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                // 获取响应体
                HttpEntity httpEntity = httpResponse.getEntity();

                String content = EntityUtils.toString(httpEntity, "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            // 关闭httpResponse
            if (httpResponse != null) {
                try {
                    httpResponse.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            // 关闭httpClient
            if (httpClient != null) {
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }




    }

}

  4. HttpPostParamTest(带参post)

    package com.xiaoge.crawler.test;

import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * @Author: 潇哥
 * @DateTime: 2020/9/21 下午4:35
 * @Description: TODO
 */
public class HttpPostParamTest {

    public static void main(String[] args) throws Exception {

        // 1. 创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 2. 创建HttpPost对象, 设置url访问地址
        HttpPost httpPost = new HttpPost("https://so.csdn.net/so/search/s.do");

        // 声明List集合, 封装表单中的参数
        List<NameValuePair> params = new ArrayList<NameValuePair>();

        // 设置请求地址是: https://so.csdn.net/so/search/s.do?q=java
        params.add(new BasicNameValuePair("q", "java"));

        // 创建表单的Entity对象, 第一个参数就是封装好的表单数据, 第二个参数就是编码
        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, "utf8");

        // 设置表单的Entity对象到Post请求中
        httpPost.setEntity(formEntity);


        CloseableHttpResponse httpResponse = null;

        try{
            // 3. 使用HttpClient发起请求, 获取response
             httpResponse = httpClient.execute(httpPost);

            // 4. 解析响应
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                // 获取响应体
                HttpEntity httpEntity = httpResponse.getEntity();

                String content = EntityUtils.toString(httpEntity, "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            // 关闭httpResponse
            if (httpResponse != null) {
                try {
                    httpResponse.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            // 关闭httpClient
            if (httpClient != null) {
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }




    }

}

  5. 连接池管理器(管理httpClient)

    package com.xiaoge.crawler.test;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @Author: 潇哥
 * @DateTime: 2020/9/21 下午5:27
 * @Description: TODO
 */
public class HttpClientPoolTest {

    public static void main(String[] args) {
        // 创建连接池管理器
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

        // 设置最大连接数
        cm.setMaxTotal(100);

        // 设置每个主机的最大连接数, (意思是每个网站只能链接10个, 因为每一个网站的域名就是一个主机)
        cm.setDefaultMaxPerRoute(10);

        // 使用连接池管理器发送请求
        doGet(cm);
        doGet(cm);

    }

    private static void doGet(PoolingHttpClientConnectionManager cm) {
        // 不是每次创建新的HttpClient, 而是从连接池中获取HttpClient对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();

        HttpGet httpGet = new HttpGet("https://www.csdn.net/");

        CloseableHttpResponse httpResponse = null;

        try {
            httpResponse = httpClient.execute(httpGet);

            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                HttpEntity httpEntity = httpResponse.getEntity();
                String content = EntityUtils.toString(httpEntity, "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {

            // 关闭httpResponse
            if (httpResponse != null) {
                try {
                    httpResponse.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            // 不能关闭HttpClient, 由连接池管理HttpClient
            // httpClient.close();
        }


    }

}

  6. HttpConfigTest(设置请求信息)

    package com.xiaoge.crawler.test;

import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * @Author: 潇哥
 * @DateTime: 2020/9/21 下午4:35
 * @Description: TODO
 */
public class HttpConfigTest {

    public static void main(String[] args) {

        // 1. 创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 2. 创建HttpGet对象, 设置url访问地址
        HttpGet httpGet = new HttpGet("https://www.csdn.net/");

        // 配置请求信息
        RequestConfig config = RequestConfig.custom().setConnectTimeout(1000)// 创建连接的最长时间, 单位是毫秒
                                        .setConnectionRequestTimeout(500) // 设置获取链接的最长时间, 单位毫秒
                                        .setSocketTimeout(10 * 1000)      // 设置数据传输的最长时间, 单位毫秒
                                        .build();

        // 给请求设置请求信息
        httpGet.setConfig(config);


        CloseableHttpResponse httpResponse = null;

        try{
            // 3. 使用HttpClient发起请求, 获取response
             httpResponse = httpClient.execute(httpGet);

            // 4. 解析响应
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                // 获取响应体
                HttpEntity httpEntity = httpResponse.getEntity();

                String content = EntityUtils.toString(httpEntity, "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            // 关闭httpResponse
            if (httpResponse != null) {
                try {
                    httpResponse.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            // 关闭httpClient
            if (httpClient != null) {
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }




    }

}
posted on 2020-09-23 20:33  gmlgxx  阅读(51)  评论(0)    收藏  举报
 
博客园  ©  2004-2025
浙公网安备 33010602011771号 浙ICP备2021040463号-3