Java爬虫入门四

使用HttpClientPool来管理连接，节省资源，类似于数据库连接池

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

/**
 * HttpClient连接池，类似数据库连接池
 *
 */
public class HttpClientPoolTest {
    public static void main(String[] args) {
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
        cm.setMaxTotal(100);// 最大连接数
        cm.setDefaultMaxPerRoute(10);// 设置每个主机的最大连接数，比如：最多允许10个连接访问百度
        doGet(cm);
    }

    private static void doGet(PoolingHttpClientConnectionManager cm) {
        CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build();
        HttpGet httpGet = new HttpGet("");
        try {
            CloseableHttpResponse response = client.execute(httpGet);
            if(response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}