网络爬虫

         //1.打开浏览器,创建HTTPClient对象对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //2.输入网址,创建get请求,创建HTTPGet对象
        HttpGet httpGet = new HttpGet("http://www.itcast.cn");

        //3.按回车,发起请求,返回响应,使用HTTPClient对象发起请求
        CloseableHttpResponse response = httpClient.execute(httpGet);

        //4.解析响应,获取数据
        //判断状态码是否是200
        if(response.getStatusLine().getStatusCode() == 200){
            HttpEntity httpEntity= response.getEntity();
            String content = EntityUtils.toString(httpEntity, "utf-8");

            System.out.println(content);
　　　　　　　}
      

 

public class HttpClientPoolTest {

    public static void main(String[] args) {
        //创建连接池管理器
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

        //设置最大连接数
        cm.setMaxTotal(100);
        //设置每个主机的最大连接数
        cm.setDefaultMaxPerRoute(10);
        
        //使用连接池管理器发起请求
        doGet(cm);
        doGet(cm);
    }

    private static void doGet(PoolingHttpClientConnectionManager cm) {
        //不是每次创建新的HTTPClient,而是从连接池中获取HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();

        HttpGet httpGet = new HttpGet("http://www.itcast.cn");

        CloseableHttpResponse response = null;
        try {
             response = httpClient.execute(httpGet);
             if(response.getStatusLine().getStatusCode() == 200){
                 String content = EntityUtils.toString(response.getEntity());
                 System.out.println(content.length());
             }

        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            if(response != null){
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            //不能关闭HTTPClient 交给连接池管理了
            //httpClient.close();
        }
    }
}

public class HttpConfigTest {

    public static void main(String[] args) {
        //创建HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //创建HttpGet对象,设置url访问地址
        HttpGet httpGet = new HttpGet("http://www.itcast.cn");

        //配置请求信息
        RequestConfig config = RequestConfig.custom().setConnectTimeout(1000) //创建连接的最长的时间,单位是毫秒
                .setConnectionRequestTimeout(500) //设置获取连接的最长时间,单位是是毫秒
                .setSocketTimeout(10 * 1000) //设置数据传输的最长时间,单位是毫秒
                .build();

        //给请求设置请求信息
        httpGet.setConfig(config);
        CloseableHttpResponse response = null;
        try {
            //使用HTTPClient发起请求,获取response
            response = httpClient.execute(httpGet);
            //解析响应
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf-8");
                System.out.println(content.length());
            }

        } catch (IOException e) {
            e.printStackTrace();

        } finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            //关闭浏览器
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }


    }
}

public class HttpGetParamTest {

    public static void main(String[] args) throws URISyntaxException {
        //创建HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //设置请求地址是:http://yun.client.itheima.com/search?keys=Java
        //创建URIBuilder
        URIBuilder uriBuilder = new URIBuilder("http://yun.client.itheima.com/search");
        //设置参数
        uriBuilder.setParameter("keys", "Java");
        //创建HttpGet对象,设置url访问地址
        HttpGet httpGet = new HttpGet(uriBuilder.build());

        System.out.println("发起请求的信息" + httpGet);

        CloseableHttpResponse response = null;
        try {
            //使用HTTPClient发起请求,获取response
             response = httpClient.execute(httpGet);
            //解析响应
            if(response.getStatusLine().getStatusCode() == 200){
                String content = EntityUtils.toString(response.getEntity(),"utf-8");
                System.out.println(content.length());
            }

        } catch (IOException e) {
            e.printStackTrace();

        }finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            //关闭浏览器
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }


    }
}

public class HttpGetTest {

    public static void main(String[] args) {
        //创建HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //创建HttpGet对象,设置url访问地址
        HttpGet httpGet = new HttpGet("http://www.itcast.cn");

        CloseableHttpResponse response = null;
        try {
            //使用HTTPClient发起请求,获取response
             response = httpClient.execute(httpGet);
            //解析响应
            if(response.getStatusLine().getStatusCode() == 200){
                String content = EntityUtils.toString(response.getEntity(),"utf-8");
                System.out.println(content.length());
            }

        } catch (IOException e) {
            e.printStackTrace();

        }finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            //关闭浏览器
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }


    }
}

public class HttpPostParamTest {

    public static void main(String[] args) throws Exception {
        //创建HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //创建HttpGet对象,设置url访问地址
        HttpPost httpPost = new HttpPost("http://yun.itheima.com/search");

        //声明List集合,封装表单中的参数
        List<NameValuePair> params = new ArrayList<NameValuePair>();
        //设置请求地址是: http
        params.add(new BasicNameValuePair("keys","Java"));
        //创建表单中的Entity对象,第一个参数是封装好的表单数据,第二个参数就是编码
        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params,"utf-8");

        //设置表单的Entity对象到POST请求中
        httpPost.setEntity(formEntity);

        CloseableHttpResponse response = null;
        try {
            //使用HTTPClient发起请求,获取response
             response = httpClient.execute(httpPost);
            //解析响应
            if(response.getStatusLine().getStatusCode() == 200){
                String content = EntityUtils.toString(response.getEntity(),"utf-8");
                System.out.println(content.length());
            }

        } catch (IOException e) {
            e.printStackTrace();

        }finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            //关闭浏览器
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }


    }
}

public class HttpPostTest {

    public static void main(String[] args) {
        //创建HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //创建HttpGet对象,设置url访问地址
        HttpPost httpPost = new HttpPost("http://www.itcast.cn");

        CloseableHttpResponse response = null;
        try {
            //使用HTTPClient发起请求,获取response
             response = httpClient.execute(httpPost);
            //解析响应
            if(response.getStatusLine().getStatusCode() == 200){
                String content = EntityUtils.toString(response.getEntity(),"utf-8");
                System.out.println(content.length());
            }

        } catch (IOException e) {
            e.printStackTrace();

        }finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            //关闭浏览器
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }


    }
}

* \* Desription: jsoup是一款Java的HTML解析器,可直接解析某个url地址,HTML文本
 * 内容.他提供了一套非常省力的api,可通过dom,css以及类似于jQuery的操作方法来取出和操作数据.
 *
 * jsoup的主要功能如下:
 * 1 从一个url,文件或字符串中解析HTML;
 * 2 使用dom或css来查找,取出数据
 * 3 可操作HTML元素,属性,文本
 */
//虽然使用jsoup可以代替HTTPClient直接发起请求解析数据,但是往往不会这样用,
// 因为在实际开发的过程中,需要
//使用到多线程,连接池,代理等等方式,
// 而jsoup对这些的支持并不是很好,
// 所以我们一般把jsoup仅仅作为HTML解析工具使用
public class JsoupFirstTest {
    @Test
    public void testUrl() throws  Exception{
        //解析url地址,第一个参数是访问的url,第二个参数是访问的超时时间
        Document doc = Jsoup.parse(new URL("http://www.itcast.cn"), 1000);

        //使用标签选择器,获取title标签中中的内容
        String title = doc.getElementsByTag("title").first().text();

        //打印
        System.out.println(title);
    }

    @Test
    public void testString() throws Exception{
        //使用工具类来读取文件,获取字符串
        String content = FileUtils.readFileToString(new File("C:\\Users\\Administrator\\Desktop\\上传视频样式代码.html"), "utf-8");

        //解析字符串
        Document doc = Jsoup.parse(content);
        String title = doc.getElementsByTag("title").first().text();
        System.out.println(title);
    }

    @Test
    public void testFile() throws Exception{
        //解析文件
        Document doc = Jsoup.parse(new File("C:\\Users\\Administrator\\Desktop\\上传视频样式代码.html"), "utf-8");
        String title = doc.getElementsByTag("title").first().text();

        System.out.println(title);
    }
}