crawler_httpurlconnection_自动编码识别

 

核心思想：

　　 1：从响应头中读取 【命中解流准确率最高】

　　 2：如果响应头中没有，打开流从源码中读取，【取舍，如果有一般在前30行会有，前100行中寻找】

       3：如果还没有，根据字节码code位置，字符识别。【前三个字符揣测】

　　 4：最终依旧没有命中采用，大陆国标编码【概率接近于0 ,gb2312】

综合效果，尚无测试到编码有问题的站点。

 

/**
     * @declare:下载 自动识别编码
     * @param url
     * @return
     * @author cphmvp
     */
    public static StringBuffer downloadHtmlAutoCode(String url) {
        StringBuffer sb = new StringBuffer();
        BufferedReader bufferReader = null;
        InputStream inputStream = null;
        BufferedInputStream bufferedInputStream = null;
        int tryNum = 0;
        while (true) {
            try {
                if (tryNum > 1) {
                    String ecodingUrl = encodParamters(url);
                    urlModel = new URL(ecodingUrl);
                } else {
                    urlModel = new URL(url);
                }
                httpURLConnection = (HttpURLConnection) urlModel
                        .openConnection();
                httpURLConnection.setConnectTimeout(connectTimeout);
                httpURLConnection.setReadTimeout(readTimeout);
                // httpURLConnection.setInstanceFollowRedirects(false);
                // httpURLConnection.setFollowRedirects(true);
                httpURLConnection
                        .setRequestProperty("User-Agent",
                                "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)");
                String redirectUrl = httpURLConnection.getURL().toString();
                if (!redirectUrl.equals(url)) {
                    LOG.info(url + "重定向后为" + redirectUrl);
                }
                // 得到响应流
                inputStream = httpURLConnection.getInputStream();
                if (null == inputStream)
                    continue;
                String charSetHeader = httpURLConnection
                        .getHeaderField("Content-Type");
                bufferedInputStream = new BufferedInputStream(inputStream);
                String charSet = null;
                // 第一步先从响应头header判断
                if (charSetHeader != null) {
                    Pattern p = Pattern.compile("charset=[\"']?(.*)['\"]?");
                    Matcher m = p.matcher(charSetHeader);
                    if (m.find()) {
                        charSet = m.group(1).trim();
                    }
                }
                // System.out.println(bufferedInputStream.available() > 0);
                // System.out.println(bufferedInputStream.markSupported());
                // 第二步 从源码中【meta http-equiv="content-type" 】判断
                // if (null == charSet) {
                // charSet = getEncode(bufferedInputStream);
                // System.out.println("---->charSet: 读流识别出来的编码" + charSet);
                // }

                // 排除非html格式 只有一两行的状况
                if (null == charSet
                        && charSetHeader.toLowerCase().contains("html")) {
                    // 缓冲区设置大些， read走的信息小于 这个值，就能reset 回来。
                    bufferedInputStream.mark(102400);
                    bufferReader = new BufferedReader(new InputStreamReader(
                            bufferedInputStream));
                    int lineNum = 1;
                    String inputLine;
                    // reset 在读至流的末尾是无法生效，故限制前100行找，找不到 放弃
                    while ((inputLine = bufferReader.readLine()) != null
                            && lineNum < 100) {
                        if (inputLine.toLowerCase().contains("charset")) {
                            charSet = RegexUtils.getString(inputLine,
                                    "charset=[\"']?(.*?)[\"']", 1);
                            LOG.info("自动识别出编码：" + charSet);
                            // 第一次匹配到后 ，不再往下判断，减少判断行数，及误判概率
                            break;
                        }
                        lineNum++;
                        inputLine = null;
                    }
                    // 第三步奏 穿插补录步奏
                    if (null == charSet) {
                        byte[] head = new byte[3];
                        bufferedInputStream.read(head);
                        if (head[0] == -1 && head[1] == -2)
                            charSet = "UTF-16";
                        if (head[0] == -2 && head[1] == -1)
                            charSet = "Unicode";
                        if (head[0] == -17 && head[1] == -69 && head[2] == -65)
                            charSet = "UTF-8";
                    }

                    // 通道回溯
                    bufferedInputStream.reset();
                }

                // 第四步奏指向默认 utf-8
                charSet = (charSet == null ? defaultEncoding : charSet);
                // 第五步奏按照正确编码解码响应流
                bufferReader = new BufferedReader(new InputStreamReader(
                        bufferedInputStream, charSet));
                String inputLine;
                while ((inputLine = bufferReader.readLine()) != null) {
                    sb.append(inputLine + "\n");
                    inputLine = null;
                }
                if (bufferReader != null)
                    try {
                        bufferReader.close();
                    } catch (IOException e) {
                        LOG.error(e);
                    }
                if (httpURLConnection != null)
                    httpURLConnection.disconnect();
                break;
            } catch (Exception e) {
                if (tryNum++ == 3) {
                    LOG.error("download page error [ " + urlModel + " ] ");
                    return null;
                }
                LOG.warn(tryNum + "次下载失败");
            }
        }
        return sb;

    }